
<!DOCTYPE html>
<html>
	<head>
		<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
		<title>job: Go Coverage Report</title>
		<style>
			body {
				background: black;
				color: rgb(80, 80, 80);
			}
			body, pre, #legend span {
				font-family: Menlo, monospace;
				font-weight: bold;
			}
			#topbar {
				background: black;
				position: fixed;
				top: 0; left: 0; right: 0;
				height: 42px;
				border-bottom: 1px solid rgb(80, 80, 80);
			}
			#content {
				margin-top: 50px;
			}
			#nav, #legend {
				float: left;
				margin-left: 10px;
			}
			#legend {
				margin-top: 12px;
			}
			#nav {
				margin-top: 10px;
			}
			#legend span {
				margin: 0 5px;
			}
			.cov0 { color: rgb(192, 0, 0) }
.cov1 { color: rgb(128, 128, 128) }
.cov2 { color: rgb(116, 140, 131) }
.cov3 { color: rgb(104, 152, 134) }
.cov4 { color: rgb(92, 164, 137) }
.cov5 { color: rgb(80, 176, 140) }
.cov6 { color: rgb(68, 188, 143) }
.cov7 { color: rgb(56, 200, 146) }
.cov8 { color: rgb(44, 212, 149) }
.cov9 { color: rgb(32, 224, 152) }
.cov10 { color: rgb(20, 236, 155) }

		</style>
	</head>
	<body>
		<div id="topbar">
			<div id="nav">
				<select id="files">
				
				<option value="file0">volcano.sh/volcano/pkg/cli/job/common.go (100.0%)</option>
				
				<option value="file1">volcano.sh/volcano/pkg/cli/job/delete.go (73.3%)</option>
				
				<option value="file2">volcano.sh/volcano/pkg/cli/job/list.go (78.8%)</option>
				
				<option value="file3">volcano.sh/volcano/pkg/cli/job/resume.go (70.0%)</option>
				
				<option value="file4">volcano.sh/volcano/pkg/cli/job/run.go (66.0%)</option>
				
				<option value="file5">volcano.sh/volcano/pkg/cli/job/suspend.go (70.0%)</option>
				
				<option value="file6">volcano.sh/volcano/pkg/cli/job/util.go (90.2%)</option>
				
				<option value="file7">volcano.sh/volcano/pkg/cli/job/view.go (90.8%)</option>
				
				<option value="file8">volcano.sh/volcano/pkg/cli/queue/common.go (100.0%)</option>
				
				<option value="file9">volcano.sh/volcano/pkg/cli/queue/create.go (83.3%)</option>
				
				<option value="file10">volcano.sh/volcano/pkg/cli/queue/delete.go (88.9%)</option>
				
				<option value="file11">volcano.sh/volcano/pkg/cli/queue/get.go (80.0%)</option>
				
				<option value="file12">volcano.sh/volcano/pkg/cli/queue/list.go (80.0%)</option>
				
				<option value="file13">volcano.sh/volcano/pkg/cli/queue/operate.go (95.5%)</option>
				
				<option value="file14">volcano.sh/volcano/pkg/cli/queue/util.go (76.9%)</option>
				
				<option value="file15">volcano.sh/volcano/pkg/cli/util/util.go (40.8%)</option>
				
				<option value="file16">volcano.sh/volcano/pkg/cli/vcancel/cancel.go (73.3%)</option>
				
				<option value="file17">volcano.sh/volcano/pkg/cli/vresume/resume.go (70.0%)</option>
				
				<option value="file18">volcano.sh/volcano/pkg/cli/vsuspend/suspend.go (70.0%)</option>
				
				<option value="file19">volcano.sh/volcano/pkg/controllers/apis/job_info.go (75.0%)</option>
				
				<option value="file20">volcano.sh/volcano/pkg/controllers/apis/request.go (100.0%)</option>
				
				<option value="file21">volcano.sh/volcano/pkg/controllers/cache/cache.go (58.2%)</option>
				
				<option value="file22">volcano.sh/volcano/pkg/controllers/garbagecollector/garbagecollector.go (37.4%)</option>
				
				<option value="file23">volcano.sh/volcano/pkg/controllers/job/job_controller.go (45.7%)</option>
				
				<option value="file24">volcano.sh/volcano/pkg/controllers/job/job_controller_actions.go (57.0%)</option>
				
				<option value="file25">volcano.sh/volcano/pkg/controllers/job/job_controller_handler.go (49.0%)</option>
				
				<option value="file26">volcano.sh/volcano/pkg/controllers/job/job_controller_plugins.go (72.2%)</option>
				
				<option value="file27">volcano.sh/volcano/pkg/controllers/job/job_controller_resync.go (3.8%)</option>
				
				<option value="file28">volcano.sh/volcano/pkg/controllers/job/job_controller_util.go (82.2%)</option>
				
				<option value="file29">volcano.sh/volcano/pkg/controllers/job/plugins/ssh/ssh.go (10.6%)</option>
				
				<option value="file30">volcano.sh/volcano/pkg/controllers/podgroup/pg_controller.go (32.5%)</option>
				
				<option value="file31">volcano.sh/volcano/pkg/controllers/podgroup/pg_controller_handler.go (51.6%)</option>
				
				<option value="file32">volcano.sh/volcano/pkg/controllers/queue/queue_controller.go (33.9%)</option>
				
				<option value="file33">volcano.sh/volcano/pkg/controllers/queue/queue_controller_action.go (25.0%)</option>
				
				<option value="file34">volcano.sh/volcano/pkg/controllers/queue/queue_controller_handler.go (57.8%)</option>
				
				<option value="file35">volcano.sh/volcano/pkg/controllers/queue/queue_controller_util.go (0.0%)</option>
				
				<option value="file36">volcano.sh/volcano/pkg/scheduler/actions/allocate/allocate.go (77.0%)</option>
				
				<option value="file37">volcano.sh/volcano/pkg/scheduler/actions/preempt/preempt.go (85.8%)</option>
				
				<option value="file38">volcano.sh/volcano/pkg/scheduler/actions/reclaim/reclaim.go (78.4%)</option>
				
				<option value="file39">volcano.sh/volcano/pkg/scheduler/api/cluster_info.go (0.0%)</option>
				
				<option value="file40">volcano.sh/volcano/pkg/scheduler/api/device_info.go (43.8%)</option>
				
				<option value="file41">volcano.sh/volcano/pkg/scheduler/api/helpers.go (40.0%)</option>
				
				<option value="file42">volcano.sh/volcano/pkg/scheduler/api/job_info.go (47.8%)</option>
				
				<option value="file43">volcano.sh/volcano/pkg/scheduler/api/namespace_info.go (83.9%)</option>
				
				<option value="file44">volcano.sh/volcano/pkg/scheduler/api/node_info.go (29.5%)</option>
				
				<option value="file45">volcano.sh/volcano/pkg/scheduler/api/numa_info.go (0.0%)</option>
				
				<option value="file46">volcano.sh/volcano/pkg/scheduler/api/pod_info.go (38.0%)</option>
				
				<option value="file47">volcano.sh/volcano/pkg/scheduler/api/queue_info.go (0.0%)</option>
				
				<option value="file48">volcano.sh/volcano/pkg/scheduler/api/resource_info.go (70.7%)</option>
				
				<option value="file49">volcano.sh/volcano/pkg/scheduler/api/silo_cluster_info.go (0.0%)</option>
				
				<option value="file50">volcano.sh/volcano/pkg/scheduler/api/test_utils.go (100.0%)</option>
				
				<option value="file51">volcano.sh/volcano/pkg/scheduler/api/types.go (25.0%)</option>
				
				<option value="file52">volcano.sh/volcano/pkg/scheduler/api/unschedule_info.go (51.9%)</option>
				
				<option value="file53">volcano.sh/volcano/pkg/scheduler/cache/cache.go (6.8%)</option>
				
				<option value="file54">volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go (29.6%)</option>
				
				<option value="file55">volcano.sh/volcano/pkg/scheduler/cache/util.go (0.0%)</option>
				
				<option value="file56">volcano.sh/volcano/pkg/scheduler/framework/arguments.go (67.6%)</option>
				
				<option value="file57">volcano.sh/volcano/pkg/scheduler/framework/framework.go (0.0%)</option>
				
				<option value="file58">volcano.sh/volcano/pkg/scheduler/framework/job_updater.go (0.0%)</option>
				
				<option value="file59">volcano.sh/volcano/pkg/scheduler/framework/plugins.go (2.7%)</option>
				
				<option value="file60">volcano.sh/volcano/pkg/scheduler/framework/session.go (0.0%)</option>
				
				<option value="file61">volcano.sh/volcano/pkg/scheduler/framework/session_plugins.go (0.0%)</option>
				
				<option value="file62">volcano.sh/volcano/pkg/scheduler/framework/statement.go (0.0%)</option>
				
				<option value="file63">volcano.sh/volcano/pkg/scheduler/plugins/binpack/binpack.go (69.6%)</option>
				
				<option value="file64">volcano.sh/volcano/pkg/scheduler/plugins/drf/drf.go (48.4%)</option>
				
				<option value="file65">volcano.sh/volcano/pkg/scheduler/plugins/numaaware/policy/factory.go (0.0%)</option>
				
				<option value="file66">volcano.sh/volcano/pkg/scheduler/plugins/numaaware/policy/policy.go (80.4%)</option>
				
				<option value="file67">volcano.sh/volcano/pkg/scheduler/plugins/numaaware/policy/policy_best_effort.go (100.0%)</option>
				
				<option value="file68">volcano.sh/volcano/pkg/scheduler/plugins/numaaware/policy/policy_none.go (0.0%)</option>
				
				<option value="file69">volcano.sh/volcano/pkg/scheduler/plugins/numaaware/policy/policy_restricted.go (100.0%)</option>
				
				<option value="file70">volcano.sh/volcano/pkg/scheduler/plugins/numaaware/policy/policy_single_numa_node.go (94.4%)</option>
				
				<option value="file71">volcano.sh/volcano/pkg/scheduler/plugins/numaaware/provider/cpumanager/cpu_assignment.go (92.4%)</option>
				
				<option value="file72">volcano.sh/volcano/pkg/scheduler/plugins/numaaware/provider/cpumanager/cpu_mng.go (77.0%)</option>
				
				<option value="file73">volcano.sh/volcano/pkg/scheduler/plugins/predicates/cache.go (3.8%)</option>
				
				<option value="file74">volcano.sh/volcano/pkg/scheduler/plugins/predicates/gpu.go (0.0%)</option>
				
				<option value="file75">volcano.sh/volcano/pkg/scheduler/plugins/predicates/predicates.go (52.0%)</option>
				
				<option value="file76">volcano.sh/volcano/pkg/scheduler/plugins/predicates/proportional.go (100.0%)</option>
				
				<option value="file77">volcano.sh/volcano/pkg/scheduler/plugins/task-topology/bucket.go (0.0%)</option>
				
				<option value="file78">volcano.sh/volcano/pkg/scheduler/plugins/task-topology/manager.go (0.0%)</option>
				
				<option value="file79">volcano.sh/volcano/pkg/scheduler/plugins/task-topology/topology.go (32.1%)</option>
				
				<option value="file80">volcano.sh/volcano/pkg/scheduler/plugins/task-topology/util.go (0.0%)</option>
				
				<option value="file81">volcano.sh/volcano/pkg/scheduler/plugins/tdm/tdm.go (68.6%)</option>
				
				<option value="file82">volcano.sh/volcano/pkg/scheduler/scheduler.go (0.0%)</option>
				
				<option value="file83">volcano.sh/volcano/pkg/scheduler/util.go (68.0%)</option>
				
				<option value="file84">volcano.sh/volcano/pkg/scheduler/util/priority_queue.go (0.0%)</option>
				
				<option value="file85">volcano.sh/volcano/pkg/scheduler/util/scheduler_helper.go (14.4%)</option>
				
				<option value="file86">volcano.sh/volcano/pkg/scheduler/util/test_utils.go (0.0%)</option>
				
				<option value="file87">volcano.sh/volcano/pkg/webhooks/admission/jobs/mutate/mutate_job.go (25.0%)</option>
				
				<option value="file88">volcano.sh/volcano/pkg/webhooks/admission/jobs/validate/admit_job.go (77.1%)</option>
				
				<option value="file89">volcano.sh/volcano/pkg/webhooks/admission/jobs/validate/util.go (97.4%)</option>
				
				<option value="file90">volcano.sh/volcano/pkg/webhooks/admission/pods/mutate/annotation.go (81.0%)</option>
				
				<option value="file91">volcano.sh/volcano/pkg/webhooks/admission/pods/mutate/factory.go (100.0%)</option>
				
				<option value="file92">volcano.sh/volcano/pkg/webhooks/admission/pods/mutate/mutate_pod.go (64.3%)</option>
				
				<option value="file93">volcano.sh/volcano/pkg/webhooks/admission/pods/mutate/namespace.go (85.7%)</option>
				
				<option value="file94">volcano.sh/volcano/pkg/webhooks/admission/pods/validate/admit_pod.go (39.7%)</option>
				
				<option value="file95">volcano.sh/volcano/pkg/webhooks/admission/queues/mutate/mutate_queue.go (88.5%)</option>
				
				<option value="file96">volcano.sh/volcano/pkg/webhooks/admission/queues/validate/validate_queue.go (95.2%)</option>
				
				</select>
			</div>
			<div id="legend">
				<span>not tracked</span>
			
				<span class="cov0">not covered</span>
				<span class="cov8">covered</span>
			
			</div>
		</div>
		<div id="content">
		
		<pre class="file" id="file0" style="display: none">/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "os"
        "path/filepath"

        "github.com/spf13/cobra"
)

type commonFlags struct {
        Master     string
        Kubeconfig string
}

func initFlags(cmd *cobra.Command, cf *commonFlags) <span class="cov8" title="1">{
        cmd.Flags().StringVarP(&amp;cf.Master, "master", "s", "", "the address of apiserver")

        kubeConfFile := os.Getenv("KUBECONFIG")
        if kubeConfFile == "" </span><span class="cov8" title="1">{
                if home := homeDir(); home != "" </span><span class="cov8" title="1">{
                        kubeConfFile = filepath.Join(home, ".kube", "config")
                }</span>
        }
        <span class="cov8" title="1">cmd.Flags().StringVarP(&amp;cf.Kubeconfig, "kubeconfig", "k", kubeConfFile, "(optional) absolute path to the kubeconfig file")</span>
}
</pre>
		
		<pre class="file" id="file1" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "volcano.sh/apis/pkg/client/clientset/versioned"
        "volcano.sh/volcano/pkg/cli/util"
)

type deleteFlags struct {
        commonFlags

        Namespace string
        JobName   string
}

var deleteJobFlags = &amp;deleteFlags{}

// InitDeleteFlags init the delete command flags.
func InitDeleteFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        initFlags(cmd, &amp;deleteJobFlags.commonFlags)

        cmd.Flags().StringVarP(&amp;deleteJobFlags.Namespace, "namespace", "n", "default", "the namespace of job")
        cmd.Flags().StringVarP(&amp;deleteJobFlags.JobName, "name", "N", "", "the name of job")
}</span>

// DeleteJob delete the job.
func DeleteJob() error <span class="cov8" title="1">{
        config, err := util.BuildConfig(deleteJobFlags.Master, deleteJobFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if deleteJobFlags.JobName == "" </span><span class="cov0" title="0">{
                err := fmt.Errorf("job name is mandatory to delete a particular job")
                return err
        }</span>

        <span class="cov8" title="1">jobClient := versioned.NewForConfigOrDie(config)
        err = jobClient.BatchV1alpha1().Jobs(deleteJobFlags.Namespace).Delete(context.TODO(), deleteJobFlags.JobName, metav1.DeleteOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>
        <span class="cov8" title="1">fmt.Printf("delete job %v successfully\n", deleteJobFlags.JobName)
        return nil</span>
}
</pre>
		
		<pre class="file" id="file2" style="display: none">/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"
        "io"
        "os"
        "strings"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
        "volcano.sh/volcano/pkg/cli/util"
)

type listFlags struct {
        commonFlags

        Namespace     string
        SchedulerName string
        allNamespace  bool
        selector      string
}

const (

        // Name  name etc below key words are used in job print format
        Name string = "Name"
        // Creation create
        Creation string = "Creation"
        // Phase phase
        Phase string = "Phase"
        // Replicas  replicas
        Replicas string = "Replicas"
        // Min  minimum
        Min string = "Min"
        // Scheduler scheduler
        Scheduler string = "Scheduler"
        // Pending  pending
        Pending string = "Pending"
        // Running running
        Running string = "Running"
        // Succeeded success
        Succeeded string = "Succeeded"
        // Terminating terminating
        Terminating string = "Terminating"
        // Version version
        Version string = "Version"
        // Failed  failed
        Failed string = "Failed"
        // Unknown pod
        Unknown string = "Unknown"
        // RetryCount retry count
        RetryCount string = "RetryCount"
        // JobType  job type
        JobType string = "JobType"
        // Namespace job namespace
        Namespace string = "Namespace"
)

var listJobFlags = &amp;listFlags{}

// InitListFlags init list command flags.
func InitListFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        initFlags(cmd, &amp;listJobFlags.commonFlags)

        cmd.Flags().StringVarP(&amp;listJobFlags.Namespace, "namespace", "n", "default", "the namespace of job")
        cmd.Flags().StringVarP(&amp;listJobFlags.SchedulerName, "scheduler", "S", "", "list job with specified scheduler name")
        cmd.Flags().BoolVarP(&amp;listJobFlags.allNamespace, "all-namespaces", "", false, "list jobs in all namespaces")
        cmd.Flags().StringVarP(&amp;listJobFlags.selector, "selector", "", "", "fuzzy matching jobName")
}</span>

// ListJobs lists all jobs details.
func ListJobs() error <span class="cov8" title="1">{
        config, err := util.BuildConfig(listJobFlags.Master, listJobFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>
        <span class="cov8" title="1">if listJobFlags.allNamespace </span><span class="cov8" title="1">{
                listJobFlags.Namespace = ""
        }</span>
        <span class="cov8" title="1">jobClient := versioned.NewForConfigOrDie(config)
        jobs, err := jobClient.BatchV1alpha1().Jobs(listJobFlags.Namespace).List(context.TODO(), metav1.ListOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if len(jobs.Items) == 0 </span><span class="cov0" title="0">{
                fmt.Printf("No resources found\n")
                return nil
        }</span>
        <span class="cov8" title="1">PrintJobs(jobs, os.Stdout)

        return nil</span>
}

// PrintJobs prints all jobs details.
func PrintJobs(jobs *v1alpha1.JobList, writer io.Writer) <span class="cov8" title="1">{
        maxLenInfo := getMaxLen(jobs)

        titleFormat := "%%-%ds%%-15s%%-12s%%-12s%%-12s%%-6s%%-10s%%-10s%%-12s%%-10s%%-12s%%-10s\n"
        contentFormat := "%%-%ds%%-15s%%-12s%%-12s%%-12d%%-6d%%-10d%%-10d%%-12d%%-10d%%-12d%%-10d\n"

        var err error
        if listJobFlags.allNamespace </span><span class="cov8" title="1">{
                _, err = fmt.Fprintf(writer, fmt.Sprintf("%%-%ds"+titleFormat, maxLenInfo[1], maxLenInfo[0]),
                        Namespace, Name, Creation, Phase, JobType, Replicas, Min, Pending, Running, Succeeded, Failed, Unknown, RetryCount)
        }</span> else<span class="cov8" title="1"> {
                _, err = fmt.Fprintf(writer, fmt.Sprintf(titleFormat, maxLenInfo[0]),
                        Name, Creation, Phase, JobType, Replicas, Min, Pending, Running, Succeeded, Failed, Unknown, RetryCount)
        }</span>
        <span class="cov8" title="1">if err != nil </span><span class="cov0" title="0">{
                fmt.Printf("Failed to print list command result: %s.\n", err)
        }</span>

        <span class="cov8" title="1">for _, job := range jobs.Items </span><span class="cov8" title="1">{
                if listJobFlags.SchedulerName != "" &amp;&amp; listJobFlags.SchedulerName != job.Spec.SchedulerName </span><span class="cov0" title="0">{
                        continue</span>
                }
                <span class="cov8" title="1">if !strings.Contains(job.Name, listJobFlags.selector) </span><span class="cov0" title="0">{
                        continue</span>
                }
                <span class="cov8" title="1">replicas := int32(0)
                for _, ts := range job.Spec.Tasks </span><span class="cov0" title="0">{
                        replicas += ts.Replicas
                }</span>
                <span class="cov8" title="1">jobType := job.ObjectMeta.Labels[v1alpha1.JobTypeKey]
                if jobType == "" </span><span class="cov8" title="1">{
                        jobType = "Batch"
                }</span>

                <span class="cov8" title="1">if listJobFlags.allNamespace </span><span class="cov8" title="1">{
                        _, err = fmt.Fprintf(writer, fmt.Sprintf("%%-%ds"+contentFormat, maxLenInfo[1], maxLenInfo[0]),
                                job.Namespace, job.Name, job.CreationTimestamp.Format("2006-01-02"), job.Status.State.Phase, jobType, replicas,
                                job.Status.MinAvailable, job.Status.Pending, job.Status.Running, job.Status.Succeeded, job.Status.Failed, job.Status.Unknown, job.Status.RetryCount)
                }</span> else<span class="cov8" title="1"> {
                        _, err = fmt.Fprintf(writer, fmt.Sprintf(contentFormat, maxLenInfo[0]),
                                job.Name, job.CreationTimestamp.Format("2006-01-02"), job.Status.State.Phase, jobType, replicas,
                                job.Status.MinAvailable, job.Status.Pending, job.Status.Running, job.Status.Succeeded, job.Status.Failed, job.Status.Unknown, job.Status.RetryCount)
                }</span>
                <span class="cov8" title="1">if err != nil </span><span class="cov0" title="0">{
                        fmt.Printf("Failed to print list command result: %s.\n", err)
                }</span>
        }
}

func getMaxLen(jobs *v1alpha1.JobList) []int <span class="cov8" title="1">{
        maxNameLen := len(Name)
        maxNamespaceLen := len(Namespace)
        for _, job := range jobs.Items </span><span class="cov8" title="1">{
                if len(job.Name) &gt; maxNameLen </span><span class="cov0" title="0">{
                        maxNameLen = len(job.Name)
                }</span>
                <span class="cov8" title="1">if len(job.Namespace) &gt; maxNamespaceLen </span><span class="cov0" title="0">{
                        maxNamespaceLen = len(job.Namespace)
                }</span>
        }

        <span class="cov8" title="1">return []int{maxNameLen + 3, maxNamespaceLen + 3}</span>
}
</pre>
		
		<pre class="file" id="file3" style="display: none">/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "fmt"

        "github.com/spf13/cobra"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/volcano/pkg/cli/util"
)

type resumeFlags struct {
        commonFlags

        Namespace string
        JobName   string
}

var resumeJobFlags = &amp;resumeFlags{}

// InitResumeFlags init resume command flags.
func InitResumeFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        initFlags(cmd, &amp;resumeJobFlags.commonFlags)

        cmd.Flags().StringVarP(&amp;resumeJobFlags.Namespace, "namespace", "n", "default", "the namespace of job")
        cmd.Flags().StringVarP(&amp;resumeJobFlags.JobName, "name", "N", "", "the name of job")
}</span>

// ResumeJob resumes the job.
func ResumeJob() error <span class="cov8" title="1">{
        config, err := util.BuildConfig(resumeJobFlags.Master, resumeJobFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>
        <span class="cov8" title="1">if resumeJobFlags.JobName == "" </span><span class="cov0" title="0">{
                err := fmt.Errorf("job name is mandatory to resume a particular job")
                return err
        }</span>

        <span class="cov8" title="1">return createJobCommand(config,
                resumeJobFlags.Namespace, resumeJobFlags.JobName,
                v1alpha1.ResumeJobAction)</span>
}
</pre>
		
		<pre class="file" id="file4" style="display: none">/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"
        "io/ioutil"
        "strings"

        "github.com/spf13/cobra"

        v1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "sigs.k8s.io/yaml"

        vcbatch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
        "volcano.sh/volcano/pkg/cli/util"
)

type runFlags struct {
        commonFlags

        Name      string
        Namespace string
        Image     string

        MinAvailable  int
        Replicas      int
        Requests      string
        Limits        string
        SchedulerName string
        FileName      string
}

var launchJobFlags = &amp;runFlags{}

// InitRunFlags init the run flags.
func InitRunFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        initFlags(cmd, &amp;launchJobFlags.commonFlags)

        cmd.Flags().StringVarP(&amp;launchJobFlags.Image, "image", "i", "busybox", "the container image of job")
        cmd.Flags().StringVarP(&amp;launchJobFlags.Namespace, "namespace", "n", "default", "the namespace of job")
        cmd.Flags().StringVarP(&amp;launchJobFlags.Name, "name", "N", "", "the name of job")
        cmd.Flags().IntVarP(&amp;launchJobFlags.MinAvailable, "min", "m", 1, "the minimal available tasks of job")
        cmd.Flags().IntVarP(&amp;launchJobFlags.Replicas, "replicas", "r", 1, "the total tasks of job")
        cmd.Flags().StringVarP(&amp;launchJobFlags.Requests, "requests", "R", "cpu=1000m,memory=100Mi", "the resource request of the task")
        cmd.Flags().StringVarP(&amp;launchJobFlags.Limits, "limits", "L", "cpu=1000m,memory=100Mi", "the resource limit of the task")
        cmd.Flags().StringVarP(&amp;launchJobFlags.SchedulerName, "scheduler", "S", "volcano", "the scheduler for this job")
        cmd.Flags().StringVarP(&amp;launchJobFlags.FileName, "filename", "f", "", "the yaml file of job")
}</span>

var jobName = "job.volcano.sh"

// RunJob creates the job.
func RunJob() error <span class="cov8" title="1">{
        config, err := util.BuildConfig(launchJobFlags.Master, launchJobFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if launchJobFlags.Name == "" &amp;&amp; launchJobFlags.FileName == "" </span><span class="cov0" title="0">{
                err = fmt.Errorf("job name cannot be left blank")
                return err
        }</span>

        <span class="cov8" title="1">req, err := populateResourceListV1(launchJobFlags.Requests)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">limit, err := populateResourceListV1(launchJobFlags.Limits)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">job, err := readFile(launchJobFlags.FileName)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if job == nil </span><span class="cov8" title="1">{
                job = constructLaunchJobFlagsJob(launchJobFlags, req, limit)
        }</span>

        <span class="cov8" title="1">jobClient := versioned.NewForConfigOrDie(config)
        newJob, err := jobClient.BatchV1alpha1().Jobs(launchJobFlags.Namespace).Create(context.TODO(), job, metav1.CreateOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if newJob.Spec.Queue == "" </span><span class="cov8" title="1">{
                newJob.Spec.Queue = "default"
        }</span>

        <span class="cov8" title="1">fmt.Printf("run job %v successfully\n", newJob.Name)

        return nil</span>
}

func readFile(filename string) (*vcbatch.Job, error) <span class="cov8" title="1">{
        if filename == "" </span><span class="cov8" title="1">{
                return nil, nil
        }</span>

        <span class="cov0" title="0">if !strings.Contains(filename, ".yaml") &amp;&amp; !strings.Contains(filename, ".yml") </span><span class="cov0" title="0">{
                return nil, fmt.Errorf("only support yaml file")
        }</span>

        <span class="cov0" title="0">file, err := ioutil.ReadFile(filename)
        if err != nil </span><span class="cov0" title="0">{
                return nil, fmt.Errorf("failed to read file, err: %v", err)
        }</span>

        <span class="cov0" title="0">var job vcbatch.Job
        if err := yaml.Unmarshal(file, &amp;job); err != nil </span><span class="cov0" title="0">{
                return nil, fmt.Errorf("failed to unmarshal file, err:  %v", err)
        }</span>

        <span class="cov0" title="0">return &amp;job, nil</span>
}

func constructLaunchJobFlagsJob(launchJobFlags *runFlags, req, limit v1.ResourceList) *vcbatch.Job <span class="cov8" title="1">{
        return &amp;vcbatch.Job{
                ObjectMeta: metav1.ObjectMeta{
                        Name:      launchJobFlags.Name,
                        Namespace: launchJobFlags.Namespace,
                },
                Spec: vcbatch.JobSpec{
                        MinAvailable:  int32(launchJobFlags.MinAvailable),
                        SchedulerName: launchJobFlags.SchedulerName,
                        Tasks: []vcbatch.TaskSpec{
                                {
                                        Replicas: int32(launchJobFlags.Replicas),

                                        Template: v1.PodTemplateSpec{
                                                ObjectMeta: metav1.ObjectMeta{
                                                        Name:   launchJobFlags.Name,
                                                        Labels: map[string]string{jobName: launchJobFlags.Name},
                                                },
                                                Spec: v1.PodSpec{
                                                        RestartPolicy: v1.RestartPolicyNever,
                                                        Containers: []v1.Container{
                                                                {
                                                                        Image:           launchJobFlags.Image,
                                                                        Name:            launchJobFlags.Name,
                                                                        ImagePullPolicy: v1.PullIfNotPresent,
                                                                        Resources: v1.ResourceRequirements{
                                                                                Limits:   limit,
                                                                                Requests: req,
                                                                        },
                                                                },
                                                        },
                                                },
                                        },
                                },
                        },
                },
        }
}</span>
</pre>
		
		<pre class="file" id="file5" style="display: none">/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "fmt"

        "github.com/spf13/cobra"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/volcano/pkg/cli/util"
)

type suspendFlags struct {
        commonFlags

        Namespace string
        JobName   string
}

var suspendJobFlags = &amp;suspendFlags{}

// InitSuspendFlags init suspend related flags.
func InitSuspendFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        initFlags(cmd, &amp;suspendJobFlags.commonFlags)

        cmd.Flags().StringVarP(&amp;suspendJobFlags.Namespace, "namespace", "n", "default", "the namespace of job")
        cmd.Flags().StringVarP(&amp;suspendJobFlags.JobName, "name", "N", "", "the name of job")
}</span>

// SuspendJob suspends the job.
func SuspendJob() error <span class="cov8" title="1">{
        config, err := util.BuildConfig(suspendJobFlags.Master, suspendJobFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if suspendJobFlags.JobName == "" </span><span class="cov0" title="0">{
                err := fmt.Errorf("job name is mandatory to suspend a particular job")
                return err
        }</span>

        <span class="cov8" title="1">return createJobCommand(config,
                suspendJobFlags.Namespace, suspendJobFlags.JobName,
                v1alpha1.AbortJobAction)</span>
}
</pre>
		
		<pre class="file" id="file6" style="display: none">/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"
        "os"
        "strings"
        "time"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/resource"
        "k8s.io/client-go/rest"

        vcbus "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

func homeDir() string <span class="cov8" title="1">{
        if h := os.Getenv("HOME"); h != "" </span><span class="cov8" title="1">{
                return h
        }</span>
        <span class="cov0" title="0">return os.Getenv("USERPROFILE")</span> // windows
}

// populateResourceListV1 takes strings of form &lt;resourceName1&gt;=&lt;value1&gt;,&lt;resourceName1&gt;=&lt;value2&gt;
// and returns ResourceList.
func populateResourceListV1(spec string) (v1.ResourceList, error) <span class="cov8" title="1">{
        // empty input gets a nil response to preserve generator test expected behaviors
        if spec == "" </span><span class="cov8" title="1">{
                return nil, nil
        }</span>

        <span class="cov8" title="1">result := v1.ResourceList{}
        resourceStatements := strings.Split(spec, ",")
        for _, resourceStatement := range resourceStatements </span><span class="cov8" title="1">{
                parts := strings.Split(resourceStatement, "=")
                if len(parts) != 2 </span><span class="cov0" title="0">{
                        return nil, fmt.Errorf("invalid argument syntax %v, expected &lt;resource&gt;=&lt;value&gt;", resourceStatement)
                }</span>
                <span class="cov8" title="1">resourceName := v1.ResourceName(parts[0])
                resourceQuantity, err := resource.ParseQuantity(parts[1])
                if err != nil </span><span class="cov0" title="0">{
                        return nil, err
                }</span>
                <span class="cov8" title="1">result[resourceName] = resourceQuantity</span>
        }
        <span class="cov8" title="1">return result, nil</span>
}

func createJobCommand(config *rest.Config, ns, name string, action vcbus.Action) error <span class="cov8" title="1">{
        jobClient := versioned.NewForConfigOrDie(config)
        job, err := jobClient.BatchV1alpha1().Jobs(ns).Get(context.TODO(), name, metav1.GetOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">ctrlRef := metav1.NewControllerRef(job, helpers.JobKind)
        cmd := &amp;vcbus.Command{
                ObjectMeta: metav1.ObjectMeta{
                        GenerateName: fmt.Sprintf("%s-%s-",
                                job.Name, strings.ToLower(string(action))),
                        Namespace: job.Namespace,
                        OwnerReferences: []metav1.OwnerReference{
                                *ctrlRef,
                        },
                },
                TargetObject: ctrlRef,
                Action:       string(action),
        }

        if _, err := jobClient.BusV1alpha1().Commands(ns).Create(context.TODO(), cmd, metav1.CreateOptions{}); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">return nil</span>
}

func translateTimestampSince(timestamp metav1.Time) string <span class="cov8" title="1">{
        if timestamp.IsZero() </span><span class="cov8" title="1">{
                return "&lt;unknown&gt;"
        }</span>
        <span class="cov8" title="1">return HumanDuration(time.Since(timestamp.Time))</span>
}

// HumanDuration translate time.Duration to human readable time string.
func HumanDuration(d time.Duration) string <span class="cov8" title="1">{
        // Allow deviation no more than 2 seconds(excluded) to tolerate machine time
        // inconsistence, it can be considered as almost now.
        if seconds := int(d.Seconds()); seconds &lt; -1 </span><span class="cov8" title="1">{
                return "&lt;invalid&gt;"
        }</span> else<span class="cov8" title="1"> if seconds &lt; 0 </span><span class="cov0" title="0">{
                return "0s"
        }</span> else<span class="cov8" title="1"> if seconds &lt; 60*2 </span><span class="cov8" title="1">{
                return fmt.Sprintf("%ds", seconds)
        }</span>
        <span class="cov8" title="1">minutes := int(d / time.Minute)
        if minutes &lt; 10 </span><span class="cov8" title="1">{
                s := int(d/time.Second) % 60
                if s == 0 </span><span class="cov8" title="1">{
                        return fmt.Sprintf("%dm", minutes)
                }</span>
                <span class="cov8" title="1">return fmt.Sprintf("%dm%ds", minutes, s)</span>
        } else<span class="cov8" title="1"> if minutes &lt; 60*3 </span><span class="cov8" title="1">{
                return fmt.Sprintf("%dm", minutes)
        }</span>
        <span class="cov8" title="1">hours := int(d / time.Hour)
        if hours &lt; 8 </span><span class="cov8" title="1">{
                m := int(d/time.Minute) % 60
                if m == 0 </span><span class="cov8" title="1">{
                        return fmt.Sprintf("%dh", hours)
                }</span>
                <span class="cov8" title="1">return fmt.Sprintf("%dh%dm", hours, m)</span>
        } else<span class="cov8" title="1"> if hours &lt; 48 </span><span class="cov8" title="1">{
                return fmt.Sprintf("%dh", hours)
        }</span> else<span class="cov8" title="1"> if hours &lt; 24*8 </span><span class="cov8" title="1">{
                h := hours % 24
                if h == 0 </span><span class="cov8" title="1">{
                        return fmt.Sprintf("%dd", hours/24)
                }</span>
                <span class="cov8" title="1">return fmt.Sprintf("%dd%dh", hours/24, h)</span>
        } else<span class="cov8" title="1"> if hours &lt; 24*365*2 </span><span class="cov8" title="1">{
                return fmt.Sprintf("%dd", hours/24)
        }</span> else<span class="cov8" title="1"> if hours &lt; 24*365*8 </span><span class="cov8" title="1">{
                return fmt.Sprintf("%dy%dd", hours/24/365, (hours/24)%365)
        }</span>
        <span class="cov8" title="1">return fmt.Sprintf("%dy", hours/24/365)</span>
}
</pre>
		
		<pre class="file" id="file7" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "encoding/json"
        "fmt"
        "io"
        "os"
        "strings"

        "github.com/spf13/cobra"

        coreV1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/client-go/kubernetes"
        "k8s.io/client-go/rest"

        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
        "volcano.sh/volcano/pkg/cli/util"
)

type viewFlags struct {
        commonFlags

        Namespace string
        JobName   string
}

// level of print indent.
const (
        Level0 = iota
        Level1
        Level2
)

var viewJobFlags = &amp;viewFlags{}

// InitViewFlags init the view command flags.
func InitViewFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        initFlags(cmd, &amp;viewJobFlags.commonFlags)

        cmd.Flags().StringVarP(&amp;viewJobFlags.Namespace, "namespace", "n", "default", "the namespace of job")
        cmd.Flags().StringVarP(&amp;viewJobFlags.JobName, "name", "N", "", "the name of job")
}</span>

// ViewJob gives full details of the job.
func ViewJob() error <span class="cov8" title="1">{
        config, err := util.BuildConfig(viewJobFlags.Master, viewJobFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>
        <span class="cov8" title="1">if viewJobFlags.JobName == "" </span><span class="cov0" title="0">{
                err := fmt.Errorf("job name (specified by --name or -N) is mandatory to view a particular job")
                return err
        }</span>

        <span class="cov8" title="1">jobClient := versioned.NewForConfigOrDie(config)
        job, err := jobClient.BatchV1alpha1().Jobs(viewJobFlags.Namespace).Get(context.TODO(), viewJobFlags.JobName, metav1.GetOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>
        <span class="cov8" title="1">if job == nil </span><span class="cov0" title="0">{
                fmt.Printf("No resources found\n")
                return nil
        }</span>
        <span class="cov8" title="1">PrintJobInfo(job, os.Stdout)
        PrintEvents(GetEvents(config, job), os.Stdout)
        return nil</span>
}

// PrintJobInfo print the job detailed info into writer.
func PrintJobInfo(job *v1alpha1.Job, writer io.Writer) <span class="cov8" title="1">{
        WriteLine(writer, Level0, "Name:       \t%s\n", job.Name)
        WriteLine(writer, Level0, "Namespace:  \t%s\n", job.Namespace)
        if len(job.Labels) &gt; 0 </span><span class="cov8" title="1">{
                label, _ := json.Marshal(job.Labels)
                WriteLine(writer, Level0, "Labels:     \t%s\n", string(label))
        }</span> else<span class="cov0" title="0"> {
                WriteLine(writer, Level0, "Labels:     \t&lt;none&gt;\n")
        }</span>
        <span class="cov8" title="1">if len(job.Annotations) &gt; 0 </span><span class="cov8" title="1">{
                annotation, _ := json.Marshal(job.Annotations)
                WriteLine(writer, Level0, "Annotations:\t%s\n", string(annotation))
        }</span> else<span class="cov0" title="0"> {
                WriteLine(writer, Level0, "Annotations:\t&lt;none&gt;\n")
        }</span>
        <span class="cov8" title="1">WriteLine(writer, Level0, "API Version:\t%s\n", job.APIVersion)
        WriteLine(writer, Level0, "Kind:       \t%s\n", job.Kind)

        WriteLine(writer, Level0, "Metadata:\n")
        WriteLine(writer, Level1, "Creation Timestamp:\t%s\n", job.CreationTimestamp)
        WriteLine(writer, Level1, "Generate Name:     \t%s\n", job.GenerateName)
        WriteLine(writer, Level1, "Generation:        \t%d\n", job.Generation)
        WriteLine(writer, Level1, "Resource Version:  \t%s\n", job.ResourceVersion)
        WriteLine(writer, Level1, "Self Link:         \t%s\n", job.SelfLink)
        WriteLine(writer, Level1, "UID:               \t%s\n", job.UID)

        WriteLine(writer, Level0, "Spec:\n")
        WriteLine(writer, Level1, "Min Available:     \t%d\n", job.Spec.MinAvailable)
        WriteLine(writer, Level1, "Plugins:\n")
        WriteLine(writer, Level2, "Env:\t%v\n", job.Spec.Plugins["env"])
        WriteLine(writer, Level2, "Ssh:\t%v\n", job.Spec.Plugins["ssh"])
        WriteLine(writer, Level1, "Scheduler Name:    \t%s\n", job.Spec.SchedulerName)
        WriteLine(writer, Level1, "Tasks:\n")
        for i := 0; i &lt; len(job.Spec.Tasks); i++ </span><span class="cov8" title="1">{
                WriteLine(writer, Level2, "Name:\t%s\n", job.Spec.Tasks[i].Name)
                WriteLine(writer, Level2, "Replicas:\t%d\n", job.Spec.Tasks[i].Replicas)
                WriteLine(writer, Level2, "Template:\n")
                WriteLine(writer, Level2+1, "Metadata:\n")
                WriteLine(writer, Level2+2, "Annotations:\n")
                WriteLine(writer, Level2+3, "Cri . Cci . Io / Container - Type:          \t%s\n", job.Spec.Tasks[i].Template.ObjectMeta.Annotations["cri.cci.io/container-type"])
                WriteLine(writer, Level2+3, "Kubernetes . Io / Availablezone:            \t%s\n", job.Spec.Tasks[i].Template.ObjectMeta.Annotations["kubernetes.io/availablezone"])
                WriteLine(writer, Level2+3, "Network . Alpha . Kubernetes . Io / Network:\t%s\n", job.Spec.Tasks[i].Template.ObjectMeta.Annotations["network.alpha.kubernetes.io/network"])
                WriteLine(writer, Level2+2, "Creation Timestamp:\t%s\n", job.Spec.Tasks[i].Template.ObjectMeta.CreationTimestamp)

                WriteLine(writer, Level2+1, "Spec:\n")
                WriteLine(writer, Level2+2, "Containers:\n")
                for j := 0; j &lt; len(job.Spec.Tasks[i].Template.Spec.Containers); j++ </span><span class="cov8" title="1">{
                        WriteLine(writer, Level2+3, "Command:\n")
                        for k := 0; k &lt; len(job.Spec.Tasks[i].Template.Spec.Containers[j].Command); k++ </span><span class="cov8" title="1">{
                                WriteLine(writer, Level2+4, "%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Command[k])
                        }</span>
                        <span class="cov8" title="1">WriteLine(writer, Level2+3, "Image:\t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Image)
                        WriteLine(writer, Level2+3, "Name: \t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Name)
                        WriteLine(writer, Level2+3, "Ports:\n")
                        for k := 0; k &lt; len(job.Spec.Tasks[i].Template.Spec.Containers[j].Ports); k++ </span><span class="cov8" title="1">{
                                WriteLine(writer, Level2+4, "Container Port:\t%d\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Ports[k].ContainerPort)
                                WriteLine(writer, Level2+4, "Name:          \t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Ports[k].Name)
                        }</span>
                        <span class="cov8" title="1">WriteLine(writer, Level2+3, "Resources:\n")
                        WriteLine(writer, Level2+4, "Limits:\n")
                        WriteLine(writer, Level2+5, "Cpu:   \t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Resources.Limits.Cpu())
                        WriteLine(writer, Level2+5, "Memory:\t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Resources.Limits.Memory())
                        WriteLine(writer, Level2+4, "Requests:\n")
                        WriteLine(writer, Level2+5, "Cpu:   \t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Resources.Requests.Cpu())
                        WriteLine(writer, Level2+5, "Memory:\t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Resources.Requests.Memory())
                        WriteLine(writer, Level2+4, "Working Dir:\t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].WorkingDir)</span>
                }
                <span class="cov8" title="1">WriteLine(writer, Level2+2, "Image Pull Secrets:\n")
                for j := 0; j &lt; len(job.Spec.Tasks[i].Template.Spec.ImagePullSecrets); j++ </span><span class="cov8" title="1">{
                        WriteLine(writer, Level2+3, "Name:     \t%s\n", job.Spec.Tasks[i].Template.Spec.ImagePullSecrets[j].Name)
                }</span>
                <span class="cov8" title="1">WriteLine(writer, Level2+2, "Restart Policy:   \t%s\n", job.Spec.Tasks[i].Template.Spec.RestartPolicy)</span>
        }

        <span class="cov8" title="1">WriteLine(writer, Level0, "Status:\n")
        if job.Status.Succeeded &gt; 0 </span><span class="cov8" title="1">{
                WriteLine(writer, Level1, "Succeeded:    \t%d\n", job.Status.Succeeded)
        }</span>
        <span class="cov8" title="1">if job.Status.Pending &gt; 0 </span><span class="cov8" title="1">{
                WriteLine(writer, Level1, "Pending:      \t%d\n", job.Status.Pending)
        }</span>
        <span class="cov8" title="1">if job.Status.Running &gt; 0 </span><span class="cov8" title="1">{
                WriteLine(writer, Level1, "Running:      \t%d\n", job.Status.Running)
        }</span>
        <span class="cov8" title="1">if job.Status.Failed &gt; 0 </span><span class="cov8" title="1">{
                WriteLine(writer, Level1, "Failed:       \t%d\n", job.Status.Failed)
        }</span>
        <span class="cov8" title="1">if job.Status.Terminating &gt; 0 </span><span class="cov8" title="1">{
                WriteLine(writer, Level1, "Terminating:  \t%d\n", job.Status.Terminating)
        }</span>
        <span class="cov8" title="1">if job.Status.Unknown &gt; 0 </span><span class="cov8" title="1">{
                WriteLine(writer, Level1, "Unknown:      \t%d\n", job.Status.Unknown)
        }</span>
        <span class="cov8" title="1">if job.Status.RetryCount &gt; 0 </span><span class="cov8" title="1">{
                WriteLine(writer, Level1, "RetryCount:   \t%d\n", job.Status.RetryCount)
        }</span>
        <span class="cov8" title="1">if job.Status.MinAvailable &gt; 0 </span><span class="cov8" title="1">{
                WriteLine(writer, Level1, "Min Available:\t%d\n", job.Status.MinAvailable)
        }</span>
        <span class="cov8" title="1">if job.Status.Version &gt; 0 </span><span class="cov8" title="1">{
                WriteLine(writer, Level1, "Version:      \t%d\n", job.Status.Version)
        }</span>

        <span class="cov8" title="1">WriteLine(writer, Level1, "State:\n")
        WriteLine(writer, Level2, "Phase:\t%s\n", job.Status.State.Phase)
        if len(job.Status.ControlledResources) &gt; 0 </span><span class="cov8" title="1">{
                WriteLine(writer, Level1, "Controlled Resources:\n")
                for key, value := range job.Status.ControlledResources </span><span class="cov8" title="1">{
                        WriteLine(writer, Level2, "%s: \t%s\n", key, value)
                }</span>
        }
}

// PrintEvents print event info to writer.
func PrintEvents(events []coreV1.Event, writer io.Writer) <span class="cov8" title="1">{
        if len(events) &gt; 0 </span><span class="cov8" title="1">{
                WriteLine(writer, Level0, "%s:\n%-15s\t%-40s\t%-30s\t%-40s\t%s\n", "Events", "Type", "Reason", "Age", "Form", "Message")
                WriteLine(writer, Level0, "%-15s\t%-40s\t%-30s\t%-40s\t%s\n", "-------", "-------", "-------", "-------", "-------")
                for _, e := range events </span><span class="cov8" title="1">{
                        var interval string
                        if e.Count &gt; 1 </span><span class="cov8" title="1">{
                                interval = fmt.Sprintf("%s (x%d over %s)", translateTimestampSince(e.LastTimestamp), e.Count, translateTimestampSince(e.FirstTimestamp))
                        }</span> else<span class="cov8" title="1"> {
                                interval = translateTimestampSince(e.FirstTimestamp)
                        }</span>
                        <span class="cov8" title="1">EventSourceString := []string{e.Source.Component}
                        if len(e.Source.Host) &gt; 0 </span><span class="cov0" title="0">{
                                EventSourceString = append(EventSourceString, e.Source.Host)
                        }</span>
                        <span class="cov8" title="1">WriteLine(writer, Level0, "%-15v\t%-40v\t%-30s\t%-40s\t%v\n",
                                e.Type,
                                e.Reason,
                                interval,
                                strings.Join(EventSourceString, ", "),
                                strings.TrimSpace(e.Message),
                        )</span>
                }
        } else<span class="cov0" title="0"> {
                WriteLine(writer, Level0, "Events: \t&lt;none&gt;\n")
        }</span>
}

// GetEvents get the job event by config.
func GetEvents(config *rest.Config, job *v1alpha1.Job) []coreV1.Event <span class="cov8" title="1">{
        kubernetes, err := kubernetes.NewForConfig(config)
        if err != nil </span><span class="cov0" title="0">{
                fmt.Printf("%v\n", err)
                return nil
        }</span>
        <span class="cov8" title="1">events, _ := kubernetes.CoreV1().Events(viewJobFlags.Namespace).List(context.TODO(), metav1.ListOptions{})
        var jobEvents []coreV1.Event
        for _, v := range events.Items </span><span class="cov8" title="1">{
                if strings.HasPrefix(v.ObjectMeta.Name, job.Name+".") </span><span class="cov8" title="1">{
                        jobEvents = append(jobEvents, v)
                }</span>
        }
        <span class="cov8" title="1">return jobEvents</span>
}

// WriteLine write lines with specified indent.
func WriteLine(writer io.Writer, spaces int, content string, params ...interface{}) <span class="cov8" title="1">{
        prefix := ""
        for i := 0; i &lt; spaces; i++ </span><span class="cov8" title="1">{
                prefix += "  "
        }</span>
        <span class="cov8" title="1">fmt.Fprintf(writer, prefix+content, params...)</span>
}
</pre>
		
		<pre class="file" id="file8" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "os"
        "path/filepath"

        "github.com/spf13/cobra"
)

type commonFlags struct {
        Master        string
        Kubeconfig    string
        SchedulerName string
}

func initFlags(cmd *cobra.Command, cf *commonFlags) <span class="cov8" title="1">{
        cmd.Flags().StringVarP(&amp;cf.SchedulerName, "scheduler", "", "volcano", "the scheduler for this job")
        cmd.Flags().StringVarP(&amp;cf.Master, "master", "s", "", "the address of apiserver")

        kubeConfFile := os.Getenv("KUBECONFIG")
        if kubeConfFile == "" </span><span class="cov8" title="1">{
                if home := homeDir(); home != "" </span><span class="cov8" title="1">{
                        kubeConfFile = filepath.Join(home, ".kube", "config")
                }</span>
        }
        <span class="cov8" title="1">cmd.Flags().StringVarP(&amp;cf.Kubeconfig, "kubeconfig", "k", kubeConfFile, "(optional) absolute path to the kubeconfig file")</span>
}
</pre>
		
		<pre class="file" id="file9" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

type createFlags struct {
        commonFlags

        Name   string
        Weight int32
        // State is state of Queue
        State string
}

var createQueueFlags = &amp;createFlags{}

// InitCreateFlags is used to init all flags during queue creating.
func InitCreateFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        initFlags(cmd, &amp;createQueueFlags.commonFlags)

        cmd.Flags().StringVarP(&amp;createQueueFlags.Name, "name", "n", "test", "the name of queue")
        cmd.Flags().Int32VarP(&amp;createQueueFlags.Weight, "weight", "w", 1, "the weight of the queue")

        cmd.Flags().StringVarP(&amp;createQueueFlags.State, "state", "S", "Open", "the state of queue")
}</span>

// CreateQueue create queue.
func CreateQueue() error <span class="cov8" title="1">{
        config, err := buildConfig(createQueueFlags.Master, createQueueFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">queue := &amp;schedulingv1beta1.Queue{
                ObjectMeta: metav1.ObjectMeta{
                        Name: createQueueFlags.Name,
                },
                Spec: schedulingv1beta1.QueueSpec{
                        Weight: createQueueFlags.Weight,
                },
                Status: schedulingv1beta1.QueueStatus{
                        State: schedulingv1beta1.QueueState(createQueueFlags.State),
                },
        }

        queueClient := versioned.NewForConfigOrDie(config)
        if _, err := queueClient.SchedulingV1beta1().Queues().Create(context.TODO(), queue, metav1.CreateOptions{}); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">return nil</span>
}
</pre>
		
		<pre class="file" id="file10" style="display: none">/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"

        "volcano.sh/apis/pkg/client/clientset/versioned"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type deleteFlags struct {
        commonFlags

        // Name is name of queue
        Name string
}

var deleteQueueFlags = &amp;deleteFlags{}

// InitDeleteFlags is used to init all flags during queue deleting.
func InitDeleteFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        initFlags(cmd, &amp;deleteQueueFlags.commonFlags)

        cmd.Flags().StringVarP(&amp;deleteQueueFlags.Name, "name", "n", "", "the name of queue")
}</span>

// DeleteQueue delete queue.
func DeleteQueue() error <span class="cov8" title="1">{
        config, err := buildConfig(deleteQueueFlags.Master, deleteQueueFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if len(deleteQueueFlags.Name) == 0 </span><span class="cov8" title="1">{
                return fmt.Errorf("queue name must be specified")
        }</span>

        <span class="cov8" title="1">queueClient := versioned.NewForConfigOrDie(config)
        return queueClient.SchedulingV1beta1().Queues().Delete(context.TODO(), deleteQueueFlags.Name, metav1.DeleteOptions{})</span>
}
</pre>
		
		<pre class="file" id="file11" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"
        "io"
        "os"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

type getFlags struct {
        commonFlags

        Name string
}

var getQueueFlags = &amp;getFlags{}

// InitGetFlags is used to init all flags.
func InitGetFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        initFlags(cmd, &amp;getQueueFlags.commonFlags)

        cmd.Flags().StringVarP(&amp;getQueueFlags.Name, "name", "n", "", "the name of queue")
}</span>

// GetQueue gets a queue.
func GetQueue() error <span class="cov8" title="1">{
        config, err := buildConfig(getQueueFlags.Master, getQueueFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if getQueueFlags.Name == "" </span><span class="cov8" title="1">{
                err := fmt.Errorf("name is mandatory to get the particular queue details")
                return err
        }</span>

        <span class="cov8" title="1">queueClient := versioned.NewForConfigOrDie(config)
        queue, err := queueClient.SchedulingV1beta1().Queues().Get(context.TODO(), getQueueFlags.Name, metav1.GetOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">PrintQueue(queue, os.Stdout)

        return nil</span>
}

// PrintQueue prints queue information.
func PrintQueue(queue *v1beta1.Queue, writer io.Writer) <span class="cov8" title="1">{
        _, err := fmt.Fprintf(writer, "%-25s%-8s%-8s%-8s%-8s%-8s%-8s\n",
                Name, Weight, State, Inqueue, Pending, Running, Unknown)
        if err != nil </span><span class="cov0" title="0">{
                fmt.Printf("Failed to print queue command result: %s.\n", err)
        }</span>
        <span class="cov8" title="1">_, err = fmt.Fprintf(writer, "%-25s%-8d%-8s%-8d%-8d%-8d%-8d\n",
                queue.Name, queue.Spec.Weight, queue.Status.State, queue.Status.Inqueue,
                queue.Status.Pending, queue.Status.Running, queue.Status.Unknown)
        if err != nil </span><span class="cov0" title="0">{
                fmt.Printf("Failed to print queue command result: %s.\n", err)
        }</span>
}
</pre>
		
		<pre class="file" id="file12" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"
        "io"
        "os"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

type listFlags struct {
        commonFlags
}

const (
        // Weight of the queue
        Weight string = "Weight"

        // Name of queue
        Name string = "Name"

        // Pending status of the queue
        Pending string = "Pending"

        // Running status of the queue
        Running string = "Running"

        // Unknown status of the queue
        Unknown string = "Unknown"

        // Inqueue status of queue
        Inqueue string = "Inqueue"

        // State is state of queue
        State string = "State"
)

var listQueueFlags = &amp;listFlags{}

// InitListFlags inits all flags.
func InitListFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        initFlags(cmd, &amp;listQueueFlags.commonFlags)
}</span>

// ListQueue lists all the queue.
func ListQueue() error <span class="cov8" title="1">{
        config, err := buildConfig(listQueueFlags.Master, listQueueFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">jobClient := versioned.NewForConfigOrDie(config)
        queues, err := jobClient.SchedulingV1beta1().Queues().List(context.TODO(), metav1.ListOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if len(queues.Items) == 0 </span><span class="cov8" title="1">{
                fmt.Printf("No resources found\n")
                return nil
        }</span>
        <span class="cov8" title="1">PrintQueues(queues, os.Stdout)

        return nil</span>
}

// PrintQueues prints queue information.
func PrintQueues(queues *v1beta1.QueueList, writer io.Writer) <span class="cov8" title="1">{
        _, err := fmt.Fprintf(writer, "%-25s%-8s%-8s%-8s%-8s%-8s%-8s\n",
                Name, Weight, State, Inqueue, Pending, Running, Unknown)
        if err != nil </span><span class="cov0" title="0">{
                fmt.Printf("Failed to print queue command result: %s.\n", err)
        }</span>
        <span class="cov8" title="1">for _, queue := range queues.Items </span><span class="cov8" title="1">{
                _, err = fmt.Fprintf(writer, "%-25s%-8d%-8s%-8d%-8d%-8d%-8d\n",
                        queue.Name, queue.Spec.Weight, queue.Status.State, queue.Status.Inqueue,
                        queue.Status.Pending, queue.Status.Running, queue.Status.Unknown)
                if err != nil </span><span class="cov0" title="0">{
                        fmt.Printf("Failed to print queue command result: %s.\n", err)
                }</span>
        }
}
</pre>
		
		<pre class="file" id="file13" style="display: none">/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"

        "github.com/spf13/cobra"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "k8s.io/apimachinery/pkg/types"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

const (
        // ActionOpen is `open` action
        ActionOpen = "open"
        // ActionClose is `close` action
        ActionClose = "close"
        // ActionUpdate is `update` action
        ActionUpdate = "update"
)

type operateFlags struct {
        commonFlags

        // Name is name of queue
        Name string
        // Weight is weight of queue
        Weight int32
        // Action is operation action of queue
        Action string
}

var operateQueueFlags = &amp;operateFlags{}

// InitOperateFlags is used to init all flags during queue operating
func InitOperateFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        initFlags(cmd, &amp;operateQueueFlags.commonFlags)

        cmd.Flags().StringVarP(&amp;operateQueueFlags.Name, "name", "n", "", "the name of queue")
        cmd.Flags().Int32VarP(&amp;operateQueueFlags.Weight, "weight", "w", 0, "the weight of the queue")
        cmd.Flags().StringVarP(&amp;operateQueueFlags.Action, "action", "a", "",
                "operate action to queue, valid actions are open, close, update")
}</span>

// OperateQueue operates queue
func OperateQueue() error <span class="cov8" title="1">{
        config, err := buildConfig(operateQueueFlags.Master, operateQueueFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if len(operateQueueFlags.Name) == 0 </span><span class="cov8" title="1">{
                return fmt.Errorf("queue name must be specified")
        }</span>

        <span class="cov8" title="1">var action v1alpha1.Action

        switch operateQueueFlags.Action </span>{
        case ActionOpen:<span class="cov8" title="1">
                action = v1alpha1.OpenQueueAction</span>
        case ActionClose:<span class="cov8" title="1">
                action = v1alpha1.CloseQueueAction</span>
        case ActionUpdate:<span class="cov8" title="1">
                if operateQueueFlags.Weight == 0 </span><span class="cov8" title="1">{
                        return fmt.Errorf("when %s queue %s, weight must be specified, "+
                                "the value must be greater than 0", ActionUpdate, operateQueueFlags.Name)
                }</span>

                <span class="cov8" title="1">queueClient := versioned.NewForConfigOrDie(config)
                patchBytes := []byte(fmt.Sprintf(`{"spec":{"weight":%d}}`, operateQueueFlags.Weight))
                _, err := queueClient.SchedulingV1beta1().Queues().Patch(context.TODO(),
                        operateQueueFlags.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{})

                return err</span>
        case "":<span class="cov8" title="1">
                return fmt.Errorf("action can not be null")</span>
        default:<span class="cov8" title="1">
                return fmt.Errorf("action %s invalid, valid actions are %s, %s and %s",
                        operateQueueFlags.Action, ActionOpen, ActionClose, ActionUpdate)</span>
        }

        <span class="cov8" title="1">return createQueueCommand(config, action)</span>
}
</pre>
		
		<pre class="file" id="file14" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"
        "os"
        "strings"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        // Initialize client auth plugin.
        _ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
        "k8s.io/client-go/rest"
        "k8s.io/client-go/tools/clientcmd"

        busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

func homeDir() string <span class="cov8" title="1">{
        if h := os.Getenv("HOME"); h != "" </span><span class="cov8" title="1">{
                return h
        }</span>
        <span class="cov0" title="0">return os.Getenv("USERPROFILE")</span> // windows
}

func buildConfig(master, kubeconfig string) (*rest.Config, error) <span class="cov8" title="1">{
        return clientcmd.BuildConfigFromFlags(master, kubeconfig)
}</span>

func createQueueCommand(config *rest.Config, action busv1alpha1.Action) error <span class="cov8" title="1">{
        queueClient := versioned.NewForConfigOrDie(config)
        queue, err := queueClient.SchedulingV1beta1().Queues().Get(context.TODO(), operateQueueFlags.Name, metav1.GetOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">ctrlRef := metav1.NewControllerRef(queue, helpers.V1beta1QueueKind)
        cmd := &amp;busv1alpha1.Command{
                ObjectMeta: metav1.ObjectMeta{
                        GenerateName: fmt.Sprintf("%s-%s-",
                                queue.Name, strings.ToLower(string(action))),
                        OwnerReferences: []metav1.OwnerReference{
                                *ctrlRef,
                        },
                },
                TargetObject: ctrlRef,
                Action:       string(action),
        }

        if _, err := queueClient.BusV1alpha1().Commands("default").Create(context.TODO(), cmd, metav1.CreateOptions{}); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">return nil</span>
}
</pre>
		
		<pre class="file" id="file15" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
        "context"
        "fmt"
        "os"
        "path/filepath"
        "strings"
        "time"

        "github.com/spf13/cobra"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/resource"
        "k8s.io/client-go/rest"
        "k8s.io/client-go/tools/clientcmd"

        vcbus "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

// CommonFlags are the flags that most command lines have.
type CommonFlags struct {
        Master     string
        Kubeconfig string
}

// InitFlags initializes the common flags for most command lines.
func InitFlags(cmd *cobra.Command, cf *CommonFlags) <span class="cov0" title="0">{
        cmd.Flags().StringVarP(&amp;cf.Master, "master", "s", "", "the address of apiserver")

        kubeConfFile := os.Getenv("KUBECONFIG")
        if kubeConfFile == "" </span><span class="cov0" title="0">{
                if home := HomeDir(); home != "" </span><span class="cov0" title="0">{
                        kubeConfFile = filepath.Join(home, ".kube", "config")
                }</span>
        }
        <span class="cov0" title="0">cmd.Flags().StringVarP(&amp;cf.Kubeconfig, "kubeconfig", "k", kubeConfFile, "(optional) absolute path to the kubeconfig file")</span>
}

// HomeDir gets the env $HOME.
func HomeDir() string <span class="cov0" title="0">{
        if h := os.Getenv("HOME"); h != "" </span><span class="cov0" title="0">{
                return h
        }</span>
        <span class="cov0" title="0">return os.Getenv("USERPROFILE")</span> // windows
}

// BuildConfig builds the configure file for command lines.
func BuildConfig(master, kubeconfig string) (*rest.Config, error) <span class="cov0" title="0">{
        return clientcmd.BuildConfigFromFlags(master, kubeconfig)
}</span>

// PopulateResourceListV1 takes strings of form &lt;resourceName1&gt;=&lt;value1&gt;,&lt;resourceName1&gt;=&lt;value2&gt; and returns ResourceList.
func PopulateResourceListV1(spec string) (v1.ResourceList, error) <span class="cov0" title="0">{
        // empty input gets a nil response to preserve generator test expected behaviors
        if spec == "" </span><span class="cov0" title="0">{
                return nil, nil
        }</span>

        <span class="cov0" title="0">result := v1.ResourceList{}
        resourceStatements := strings.Split(spec, ",")
        for _, resourceStatement := range resourceStatements </span><span class="cov0" title="0">{
                parts := strings.Split(resourceStatement, "=")
                if len(parts) != 2 </span><span class="cov0" title="0">{
                        return nil, fmt.Errorf("invalid argument syntax %v, expected &lt;resource&gt;=&lt;value&gt;", resourceStatement)
                }</span>
                <span class="cov0" title="0">resourceName := v1.ResourceName(parts[0])
                resourceQuantity, err := resource.ParseQuantity(parts[1])
                if err != nil </span><span class="cov0" title="0">{
                        return nil, err
                }</span>
                <span class="cov0" title="0">result[resourceName] = resourceQuantity</span>
        }
        <span class="cov0" title="0">return result, nil</span>
}

// CreateQueueCommand executes a command such as open/close
func CreateQueueCommand(vcClient *versioned.Clientset, ns, name string, action vcbus.Action) error <span class="cov0" title="0">{
        queue, err := vcClient.SchedulingV1beta1().Queues().Get(context.TODO(), name, metav1.GetOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>
        <span class="cov0" title="0">ctrlRef := metav1.NewControllerRef(queue, helpers.V1beta1QueueKind)
        cmd := &amp;vcbus.Command{
                ObjectMeta: metav1.ObjectMeta{
                        GenerateName: fmt.Sprintf("%s-%s-",
                                queue.Name, strings.ToLower(string(action))),
                        Namespace: queue.Namespace,
                        OwnerReferences: []metav1.OwnerReference{
                                *ctrlRef,
                        },
                },
                TargetObject: ctrlRef,
                Action:       string(action),
        }

        if _, err := vcClient.BusV1alpha1().Commands(ns).Create(context.TODO(), cmd, metav1.CreateOptions{}); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">return nil</span>
}

// CreateJobCommand executes a command such as resume/suspend.
func CreateJobCommand(config *rest.Config, ns, name string, action vcbus.Action) error <span class="cov0" title="0">{
        jobClient := versioned.NewForConfigOrDie(config)
        job, err := jobClient.BatchV1alpha1().Jobs(ns).Get(context.TODO(), name, metav1.GetOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">ctrlRef := metav1.NewControllerRef(job, helpers.JobKind)
        cmd := &amp;vcbus.Command{
                ObjectMeta: metav1.ObjectMeta{
                        GenerateName: fmt.Sprintf("%s-%s-",
                                job.Name, strings.ToLower(string(action))),
                        Namespace: job.Namespace,
                        OwnerReferences: []metav1.OwnerReference{
                                *ctrlRef,
                        },
                },
                TargetObject: ctrlRef,
                Action:       string(action),
        }

        if _, err := jobClient.BusV1alpha1().Commands(ns).Create(context.TODO(), cmd, metav1.CreateOptions{}); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">return nil</span>
}

// TranslateTimestampSince translates the time stamp.
func TranslateTimestampSince(timestamp metav1.Time) string <span class="cov0" title="0">{
        if timestamp.IsZero() </span><span class="cov0" title="0">{
                return "&lt;unknown&gt;"
        }</span>
        <span class="cov0" title="0">return HumanDuration(time.Since(timestamp.Time))</span>
}

// HumanDuration translate time.Duration to human readable time string.
func HumanDuration(d time.Duration) string <span class="cov8" title="1">{
        // Allow deviation no more than 2 seconds(excluded) to tolerate machine time
        // inconsistence, it can be considered as almost now.
        if seconds := int(d.Seconds()); seconds &lt; -1 </span><span class="cov8" title="1">{
                return "&lt;invalid&gt;"
        }</span> else<span class="cov8" title="1"> if seconds &lt; 0 </span><span class="cov0" title="0">{
                return "0s"
        }</span> else<span class="cov8" title="1"> if seconds &lt; 60*2 </span><span class="cov8" title="1">{
                return fmt.Sprintf("%ds", seconds)
        }</span>
        <span class="cov8" title="1">minutes := int(d / time.Minute)
        if minutes &lt; 10 </span><span class="cov8" title="1">{
                s := int(d/time.Second) % 60
                if s == 0 </span><span class="cov8" title="1">{
                        return fmt.Sprintf("%dm", minutes)
                }</span>
                <span class="cov8" title="1">return fmt.Sprintf("%dm%ds", minutes, s)</span>
        } else<span class="cov8" title="1"> if minutes &lt; 60*3 </span><span class="cov8" title="1">{
                return fmt.Sprintf("%dm", minutes)
        }</span>
        <span class="cov8" title="1">hours := int(d / time.Hour)
        if hours &lt; 8 </span><span class="cov8" title="1">{
                m := int(d/time.Minute) % 60
                if m == 0 </span><span class="cov8" title="1">{
                        return fmt.Sprintf("%dh", hours)
                }</span>
                <span class="cov8" title="1">return fmt.Sprintf("%dh%dm", hours, m)</span>
        } else<span class="cov8" title="1"> if hours &lt; 48 </span><span class="cov8" title="1">{
                return fmt.Sprintf("%dh", hours)
        }</span> else<span class="cov8" title="1"> if hours &lt; 24*8 </span><span class="cov8" title="1">{
                h := hours % 24
                if h == 0 </span><span class="cov8" title="1">{
                        return fmt.Sprintf("%dd", hours/24)
                }</span>
                <span class="cov8" title="1">return fmt.Sprintf("%dd%dh", hours/24, h)</span>
        } else<span class="cov8" title="1"> if hours &lt; 24*365*2 </span><span class="cov8" title="1">{
                return fmt.Sprintf("%dd", hours/24)
        }</span> else<span class="cov8" title="1"> if hours &lt; 24*365*8 </span><span class="cov8" title="1">{
                return fmt.Sprintf("%dy%dd", hours/24/365, (hours/24)%365)
        }</span>
        <span class="cov8" title="1">return fmt.Sprintf("%dy", hours/24/365)</span>
}
</pre>
		
		<pre class="file" id="file16" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vcancel

import (
        "context"
        "fmt"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "volcano.sh/apis/pkg/client/clientset/versioned"
        "volcano.sh/volcano/pkg/cli/util"
)

type cancelFlags struct {
        util.CommonFlags

        Namespace string
        JobName   string
}

var cancelJobFlags = &amp;cancelFlags{}

// InitCancelFlags init the cancel command flags.
func InitCancelFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        util.InitFlags(cmd, &amp;cancelJobFlags.CommonFlags)

        cmd.Flags().StringVarP(&amp;cancelJobFlags.Namespace, "namespace", "N", "default", "the namespace of job")
        cmd.Flags().StringVarP(&amp;cancelJobFlags.JobName, "name", "n", "", "the name of job")
}</span>

// CancelJob cancel the job.
func CancelJob() error <span class="cov8" title="1">{
        config, err := util.BuildConfig(cancelJobFlags.Master, cancelJobFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if cancelJobFlags.JobName == "" </span><span class="cov0" title="0">{
                err := fmt.Errorf("job name is mandatory to cancel a particular job")
                return err
        }</span>

        <span class="cov8" title="1">jobClient := versioned.NewForConfigOrDie(config)
        err = jobClient.BatchV1alpha1().Jobs(cancelJobFlags.Namespace).Delete(context.TODO(), cancelJobFlags.JobName, metav1.DeleteOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>
        <span class="cov8" title="1">fmt.Printf("cancel job %v successfully\n", cancelJobFlags.JobName)
        return nil</span>
}
</pre>
		
		<pre class="file" id="file17" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vresume

import (
        "fmt"

        "github.com/spf13/cobra"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/volcano/pkg/cli/util"
)

type resumeFlags struct {
        util.CommonFlags

        Namespace string
        JobName   string
}

var resumeJobFlags = &amp;resumeFlags{}

// InitResumeFlags init resume command flags.
func InitResumeFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        util.InitFlags(cmd, &amp;resumeJobFlags.CommonFlags)

        cmd.Flags().StringVarP(&amp;resumeJobFlags.Namespace, "namespace", "N", "default", "the namespace of job")
        cmd.Flags().StringVarP(&amp;resumeJobFlags.JobName, "name", "n", "", "the name of job")
}</span>

// ResumeJob resumes the job.
func ResumeJob() error <span class="cov8" title="1">{
        config, err := util.BuildConfig(resumeJobFlags.Master, resumeJobFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>
        <span class="cov8" title="1">if resumeJobFlags.JobName == "" </span><span class="cov0" title="0">{
                err := fmt.Errorf("job name is mandatory to resume a particular job")
                return err
        }</span>

        <span class="cov8" title="1">return util.CreateJobCommand(config,
                resumeJobFlags.Namespace, resumeJobFlags.JobName,
                v1alpha1.ResumeJobAction)</span>
}
</pre>
		
		<pre class="file" id="file18" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vsuspend

import (
        "fmt"

        "github.com/spf13/cobra"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/volcano/pkg/cli/util"
)

type suspendFlags struct {
        util.CommonFlags

        Namespace string
        JobName   string
}

var suspendJobFlags = &amp;suspendFlags{}

// InitSuspendFlags init suspend related flags.
func InitSuspendFlags(cmd *cobra.Command) <span class="cov8" title="1">{
        util.InitFlags(cmd, &amp;suspendJobFlags.CommonFlags)

        cmd.Flags().StringVarP(&amp;suspendJobFlags.Namespace, "namespace", "N", "default", "the namespace of job")
        cmd.Flags().StringVarP(&amp;suspendJobFlags.JobName, "name", "n", "", "the name of job")
}</span>

// SuspendJob suspends the job.
func SuspendJob() error <span class="cov8" title="1">{
        config, err := util.BuildConfig(suspendJobFlags.Master, suspendJobFlags.Kubeconfig)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if suspendJobFlags.JobName == "" </span><span class="cov0" title="0">{
                err := fmt.Errorf("job name is mandatory to suspend a particular job")
                return err
        }</span>

        <span class="cov8" title="1">return util.CreateJobCommand(config,
                suspendJobFlags.Namespace, suspendJobFlags.JobName,
                v1alpha1.AbortJobAction)</span>
}
</pre>
		
		<pre class="file" id="file19" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apis

import (
        "fmt"

        v1 "k8s.io/api/core/v1"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
)

//JobInfo struct.
type JobInfo struct {
        Namespace string
        Name      string

        Job  *batch.Job
        Pods map[string]map[string]*v1.Pod
}

//Clone function clones the k8s pod values to the JobInfo struct.
func (ji *JobInfo) Clone() *JobInfo <span class="cov8" title="1">{
        job := &amp;JobInfo{
                Namespace: ji.Namespace,
                Name:      ji.Name,
                Job:       ji.Job,

                Pods: make(map[string]map[string]*v1.Pod),
        }

        for key, pods := range ji.Pods </span><span class="cov0" title="0">{
                job.Pods[key] = make(map[string]*v1.Pod)
                for pn, pod := range pods </span><span class="cov0" title="0">{
                        job.Pods[key][pn] = pod
                }</span>
        }

        <span class="cov8" title="1">return job</span>
}

//SetJob sets the volcano jobs values to the JobInfo struct.
func (ji *JobInfo) SetJob(job *batch.Job) <span class="cov8" title="1">{
        ji.Name = job.Name
        ji.Namespace = job.Namespace
        ji.Job = job
}</span>

//AddPod adds the k8s pod object values to the Pods field
//of JobStruct if it doesn't exist. Otherwise it throws error.
func (ji *JobInfo) AddPod(pod *v1.Pod) error <span class="cov8" title="1">{
        taskName, found := pod.Annotations[batch.TaskSpecKey]
        if !found </span><span class="cov0" title="0">{
                return fmt.Errorf("failed to taskName of Pod &lt;%s/%s&gt;",
                        pod.Namespace, pod.Name)
        }</span>

        <span class="cov8" title="1">_, found = pod.Annotations[batch.JobVersion]
        if !found </span><span class="cov0" title="0">{
                return fmt.Errorf("failed to find jobVersion of Pod &lt;%s/%s&gt;",
                        pod.Namespace, pod.Name)
        }</span>

        <span class="cov8" title="1">if _, found := ji.Pods[taskName]; !found </span><span class="cov8" title="1">{
                ji.Pods[taskName] = make(map[string]*v1.Pod)
        }</span>
        <span class="cov8" title="1">if _, found := ji.Pods[taskName][pod.Name]; found </span><span class="cov8" title="1">{
                return fmt.Errorf("duplicated pod")
        }</span>
        <span class="cov8" title="1">ji.Pods[taskName][pod.Name] = pod

        return nil</span>
}

//UpdatePod updates the k8s pod object values to the existing pod.
func (ji *JobInfo) UpdatePod(pod *v1.Pod) error <span class="cov8" title="1">{
        taskName, found := pod.Annotations[batch.TaskSpecKey]
        if !found </span><span class="cov0" title="0">{
                return fmt.Errorf("failed to find taskName of Pod &lt;%s/%s&gt;",
                        pod.Namespace, pod.Name)
        }</span>
        <span class="cov8" title="1">_, found = pod.Annotations[batch.JobVersion]
        if !found </span><span class="cov0" title="0">{
                return fmt.Errorf("failed to find jobVersion of Pod &lt;%s/%s&gt;",
                        pod.Namespace, pod.Name)
        }</span>

        <span class="cov8" title="1">if _, found := ji.Pods[taskName]; !found </span><span class="cov0" title="0">{
                return fmt.Errorf("can not find task %s in cache", taskName)
        }</span>
        <span class="cov8" title="1">if _, found := ji.Pods[taskName][pod.Name]; !found </span><span class="cov0" title="0">{
                return fmt.Errorf("can not find pod &lt;%s/%s&gt; in cache",
                        pod.Namespace, pod.Name)
        }</span>
        <span class="cov8" title="1">ji.Pods[taskName][pod.Name] = pod

        return nil</span>
}

//DeletePod deletes the given k8s pod from the JobInfo struct.
func (ji *JobInfo) DeletePod(pod *v1.Pod) error <span class="cov8" title="1">{
        taskName, found := pod.Annotations[batch.TaskSpecKey]
        if !found </span><span class="cov0" title="0">{
                return fmt.Errorf("failed to find taskName of Pod &lt;%s/%s&gt;",
                        pod.Namespace, pod.Name)
        }</span>
        <span class="cov8" title="1">_, found = pod.Annotations[batch.JobVersion]
        if !found </span><span class="cov0" title="0">{
                return fmt.Errorf("failed to find jobVersion of Pod &lt;%s/%s&gt;",
                        pod.Namespace, pod.Name)
        }</span>

        <span class="cov8" title="1">if pods, found := ji.Pods[taskName]; found </span><span class="cov8" title="1">{
                delete(pods, pod.Name)
                if len(pods) == 0 </span><span class="cov8" title="1">{
                        delete(ji.Pods, taskName)
                }</span>
        }

        <span class="cov8" title="1">return nil</span>
}
</pre>
		
		<pre class="file" id="file20" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apis

import (
        "fmt"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
)

//Request struct.
type Request struct {
        Namespace string
        JobName   string
        TaskName  string
        QueueName string

        Event      v1alpha1.Event
        ExitCode   int32
        Action     v1alpha1.Action
        JobVersion int32
}

// String function returns the request in string format.
func (r Request) String() string <span class="cov8" title="1">{
        return fmt.Sprintf(
                "Queue: %s, Job: %s/%s, Task:%s, Event:%s, ExitCode:%d, Action:%s, JobVersion: %d",
                r.QueueName, r.Namespace, r.JobName, r.TaskName, r.Event, r.ExitCode, r.Action, r.JobVersion)
}</span>
</pre>
		
		<pre class="file" id="file21" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
        "fmt"
        "sync"
        "time"

        "golang.org/x/time/rate"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/util/wait"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/volcano/pkg/controllers/apis"
)

type jobCache struct {
        sync.Mutex

        jobs        map[string]*apis.JobInfo
        deletedJobs workqueue.RateLimitingInterface
}

func keyFn(ns, name string) string <span class="cov8" title="1">{
        return fmt.Sprintf("%s/%s", ns, name)
}</span>

//JobKeyByName gets the key for the job name.
func JobKeyByName(namespace string, name string) string <span class="cov0" title="0">{
        return keyFn(namespace, name)
}</span>

//JobKeyByReq gets the key for the job request.
func JobKeyByReq(req *apis.Request) string <span class="cov0" title="0">{
        return keyFn(req.Namespace, req.JobName)
}</span>

//JobKey gets the "ns"/"name" format of the given job.
func JobKey(job *v1alpha1.Job) string <span class="cov8" title="1">{
        return keyFn(job.Namespace, job.Name)
}</span>

func jobTerminated(job *apis.JobInfo) bool <span class="cov0" title="0">{
        return job.Job == nil &amp;&amp; len(job.Pods) == 0
}</span>

func jobKeyOfPod(pod *v1.Pod) (string, error) <span class="cov8" title="1">{
        jobName, found := pod.Annotations[v1alpha1.JobNameKey]
        if !found </span><span class="cov8" title="1">{
                return "", fmt.Errorf("failed to find job name of pod &lt;%s/%s&gt;",
                        pod.Namespace, pod.Name)
        }</span>

        <span class="cov8" title="1">return keyFn(pod.Namespace, jobName), nil</span>
}

// New gets the job Cache.
func New() Cache <span class="cov8" title="1">{
        queue := workqueue.NewMaxOfRateLimiter(
                workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 180*time.Second),
                // 10 qps, 100 bucket size.  This is only for retry speed and its only the overall factor (not per item)
                &amp;workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
        )

        return &amp;jobCache{
                jobs:        map[string]*apis.JobInfo{},
                deletedJobs: workqueue.NewRateLimitingQueue(queue),
        }
}</span>

func (jc *jobCache) Get(key string) (*apis.JobInfo, error) <span class="cov8" title="1">{
        jc.Lock()
        defer jc.Unlock()

        job, found := jc.jobs[key]
        if !found </span><span class="cov8" title="1">{
                return nil, fmt.Errorf("failed to find job &lt;%s&gt;", key)
        }</span>

        <span class="cov8" title="1">if job.Job == nil </span><span class="cov8" title="1">{
                return nil, fmt.Errorf("job &lt;%s&gt; is not ready", key)
        }</span>

        <span class="cov8" title="1">return job.Clone(), nil</span>
}

func (jc *jobCache) GetStatus(key string) (*v1alpha1.JobStatus, error) <span class="cov8" title="1">{
        jc.Lock()
        defer jc.Unlock()

        job, found := jc.jobs[key]
        if !found </span><span class="cov8" title="1">{
                return nil, fmt.Errorf("failed to find job &lt;%s&gt;", key)
        }</span>

        <span class="cov8" title="1">if job.Job == nil </span><span class="cov0" title="0">{
                return nil, fmt.Errorf("job &lt;%s&gt; is not ready", key)
        }</span>

        <span class="cov8" title="1">status := job.Job.Status

        return &amp;status, nil</span>
}

func (jc *jobCache) Add(job *v1alpha1.Job) error <span class="cov8" title="1">{
        jc.Lock()
        defer jc.Unlock()
        key := JobKey(job)
        if jobInfo, found := jc.jobs[key]; found </span><span class="cov8" title="1">{
                if jobInfo.Job == nil </span><span class="cov0" title="0">{
                        jobInfo.SetJob(job)

                        return nil
                }</span>
                <span class="cov8" title="1">return fmt.Errorf("duplicated jobInfo &lt;%v&gt;", key)</span>
        }

        <span class="cov8" title="1">jc.jobs[key] = &amp;apis.JobInfo{
                Name:      job.Name,
                Namespace: job.Namespace,

                Job:  job,
                Pods: make(map[string]map[string]*v1.Pod),
        }

        return nil</span>
}

func (jc *jobCache) Update(obj *v1alpha1.Job) error <span class="cov8" title="1">{
        jc.Lock()
        defer jc.Unlock()

        key := JobKey(obj)
        job, found := jc.jobs[key]
        if !found </span><span class="cov8" title="1">{
                return fmt.Errorf("failed to find job &lt;%v&gt;", key)
        }</span>
        <span class="cov8" title="1">job.Job = obj

        return nil</span>
}

func (jc *jobCache) Delete(obj *v1alpha1.Job) error <span class="cov8" title="1">{
        jc.Lock()
        defer jc.Unlock()

        key := JobKey(obj)
        jobInfo, found := jc.jobs[key]
        if !found </span><span class="cov8" title="1">{
                return fmt.Errorf("failed to find job &lt;%v&gt;", key)
        }</span>
        <span class="cov8" title="1">jobInfo.Job = nil
        jc.deleteJob(jobInfo)

        return nil</span>
}

func (jc *jobCache) AddPod(pod *v1.Pod) error <span class="cov8" title="1">{
        jc.Lock()
        defer jc.Unlock()

        key, err := jobKeyOfPod(pod)
        if err != nil </span><span class="cov8" title="1">{
                return err
        }</span>

        <span class="cov8" title="1">job, found := jc.jobs[key]
        if !found </span><span class="cov0" title="0">{
                job = &amp;apis.JobInfo{
                        Pods: make(map[string]map[string]*v1.Pod),
                }
                jc.jobs[key] = job
        }</span>

        <span class="cov8" title="1">return job.AddPod(pod)</span>
}

func (jc *jobCache) UpdatePod(pod *v1.Pod) error <span class="cov8" title="1">{
        jc.Lock()
        defer jc.Unlock()

        key, err := jobKeyOfPod(pod)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">job, found := jc.jobs[key]
        if !found </span><span class="cov0" title="0">{
                job = &amp;apis.JobInfo{
                        Pods: make(map[string]map[string]*v1.Pod),
                }
                jc.jobs[key] = job
        }</span>

        <span class="cov8" title="1">return job.UpdatePod(pod)</span>
}

func (jc *jobCache) DeletePod(pod *v1.Pod) error <span class="cov8" title="1">{
        jc.Lock()
        defer jc.Unlock()

        key, err := jobKeyOfPod(pod)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">job, found := jc.jobs[key]
        if !found </span><span class="cov0" title="0">{
                job = &amp;apis.JobInfo{
                        Pods: make(map[string]map[string]*v1.Pod),
                }
                jc.jobs[key] = job
        }</span>

        <span class="cov8" title="1">if err := job.DeletePod(pod); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if jc.jobs[key].Job == nil </span><span class="cov0" title="0">{
                jc.deleteJob(job)
        }</span>

        <span class="cov8" title="1">return nil</span>
}

func (jc *jobCache) Run(stopCh &lt;-chan struct{}) <span class="cov0" title="0">{
        wait.Until(jc.worker, 0, stopCh)
}</span>

func (jc *jobCache) TaskCompleted(jobKey, taskName string) bool <span class="cov8" title="1">{
        jc.Lock()
        defer jc.Unlock()

        var taskReplicas, completed int32

        jobInfo, found := jc.jobs[jobKey]
        if !found </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov8" title="1">taskPods, found := jobInfo.Pods[taskName]

        if !found </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov8" title="1">if jobInfo.Job == nil </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov8" title="1">for _, task := range jobInfo.Job.Spec.Tasks </span><span class="cov8" title="1">{
                if task.Name == taskName </span><span class="cov8" title="1">{
                        taskReplicas = task.Replicas
                        break</span>
                }
        }
        <span class="cov8" title="1">if taskReplicas &lt;= 0 </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov8" title="1">for _, pod := range taskPods </span><span class="cov8" title="1">{
                if pod.Status.Phase == v1.PodSucceeded </span><span class="cov8" title="1">{
                        completed++
                }</span>
        }
        <span class="cov8" title="1">return completed &gt;= taskReplicas</span>
}

func (jc *jobCache) TaskFailed(jobKey, taskName string) bool <span class="cov0" title="0">{
        jc.Lock()
        defer jc.Unlock()

        var taskReplicas, retried, maxRetry int32

        jobInfo, found := jc.jobs[jobKey]
        if !found </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov0" title="0">taskPods, found := jobInfo.Pods[taskName]

        if !found || jobInfo.Job == nil </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov0" title="0">for _, task := range jobInfo.Job.Spec.Tasks </span><span class="cov0" title="0">{
                if task.Name == taskName </span><span class="cov0" title="0">{
                        maxRetry = task.MaxRetry
                        taskReplicas = task.Replicas
                        break</span>
                }
        }

        // maxRetry == -1 means no limit
        <span class="cov0" title="0">if taskReplicas == 0 || maxRetry == -1 </span><span class="cov0" title="0">{
                return false
        }</span>

        // Compatible with existing job
        <span class="cov0" title="0">if maxRetry == 0 </span><span class="cov0" title="0">{
                maxRetry = 3
        }</span>

        <span class="cov0" title="0">for _, pod := range taskPods </span><span class="cov0" title="0">{
                if pod.Status.Phase == v1.PodRunning || pod.Status.Phase == v1.PodPending </span><span class="cov0" title="0">{
                        for j := range pod.Status.InitContainerStatuses </span><span class="cov0" title="0">{
                                stat := pod.Status.InitContainerStatuses[j]
                                retried += stat.RestartCount
                        }</span>
                        <span class="cov0" title="0">for j := range pod.Status.ContainerStatuses </span><span class="cov0" title="0">{
                                stat := pod.Status.ContainerStatuses[j]
                                retried += stat.RestartCount
                        }</span>
                }
        }
        <span class="cov0" title="0">return retried &gt; maxRetry</span>
}

func (jc *jobCache) worker() <span class="cov0" title="0">{
        for jc.processCleanupJob() </span>{<span class="cov0" title="0">
        }</span>
}

func (jc *jobCache) processCleanupJob() bool <span class="cov0" title="0">{
        obj, shutdown := jc.deletedJobs.Get()
        if shutdown </span><span class="cov0" title="0">{
                return false
        }</span>
        <span class="cov0" title="0">defer jc.deletedJobs.Done(obj)

        job, ok := obj.(*apis.JobInfo)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("failed to convert %v to *apis.JobInfo", obj)
                return true
        }</span>

        <span class="cov0" title="0">jc.Mutex.Lock()
        defer jc.Mutex.Unlock()

        if jobTerminated(job) </span><span class="cov0" title="0">{
                jc.deletedJobs.Forget(obj)
                key := keyFn(job.Namespace, job.Name)
                delete(jc.jobs, key)
                klog.V(3).Infof("Job &lt;%s&gt; was deleted.", key)
        }</span> else<span class="cov0" title="0"> {
                // Retry
                jc.deleteJob(job)
        }</span>
        <span class="cov0" title="0">return true</span>
}

func (jc *jobCache) deleteJob(job *apis.JobInfo) <span class="cov8" title="1">{
        klog.V(3).Infof("Try to delete Job &lt;%v/%v&gt;",
                job.Namespace, job.Name)

        jc.deletedJobs.AddRateLimited(job)
}</span>
</pre>
		
		<pre class="file" id="file22" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package garbagecollector

import (
        "context"
        "fmt"
        "time"

        "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/util/wait"
        "k8s.io/client-go/tools/cache"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        vcclientset "volcano.sh/apis/pkg/client/clientset/versioned"
        informerfactory "volcano.sh/apis/pkg/client/informers/externalversions"
        batchinformers "volcano.sh/apis/pkg/client/informers/externalversions/batch/v1alpha1"
        batchlisters "volcano.sh/apis/pkg/client/listers/batch/v1alpha1"
        "volcano.sh/volcano/pkg/controllers/framework"
)

func init() <span class="cov8" title="1">{
        framework.RegisterController(&amp;gccontroller{})
}</span>

// gccontroller runs reflectors to watch for changes of managed API
// objects. Currently it only watches Jobs. Triggered by Job creation
// and updates, it enqueues Jobs that have non-nil `.spec.ttlSecondsAfterFinished`
// to the `queue`. The gccontroller has workers who consume `queue`, check whether
// the Job TTL has expired or not; if the Job TTL hasn't expired, it will add the
// Job to the queue after the TTL is expected to expire; if the TTL has expired, the
// worker will send requests to the API server to delete the Jobs accordingly.
// This is implemented outside of Job controller for separation of concerns, and
// because it will be extended to handle other finishable resource types.
type gccontroller struct {
        vcClient vcclientset.Interface

        jobInformer batchinformers.JobInformer

        // A store of jobs
        jobLister batchlisters.JobLister
        jobSynced func() bool

        // queues that need to be updated.
        queue workqueue.RateLimitingInterface
}

func (gc *gccontroller) Name() string <span class="cov8" title="1">{
        return "gc-controller"
}</span>

// Initialize creates an instance of gccontroller.
func (gc *gccontroller) Initialize(opt *framework.ControllerOption) error <span class="cov8" title="1">{
        gc.vcClient = opt.VolcanoClient
        jobInformer := informerfactory.NewSharedInformerFactory(gc.vcClient, 0).Batch().V1alpha1().Jobs()

        gc.jobInformer = jobInformer
        gc.jobLister = jobInformer.Lister()
        gc.jobSynced = jobInformer.Informer().HasSynced
        gc.queue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())

        jobInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    gc.addJob,
                UpdateFunc: gc.updateJob,
        })

        return nil
}</span>

// Run starts the worker to clean up Jobs.
func (gc *gccontroller) Run(stopCh &lt;-chan struct{}) <span class="cov0" title="0">{
        defer gc.queue.ShutDown()

        klog.Infof("Starting garbage collector")
        defer klog.Infof("Shutting down garbage collector")

        go gc.jobInformer.Informer().Run(stopCh)
        if !cache.WaitForCacheSync(stopCh, gc.jobSynced) </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov0" title="0">go wait.Until(gc.worker, time.Second, stopCh)

        &lt;-stopCh</span>
}

func (gc *gccontroller) addJob(obj interface{}) <span class="cov0" title="0">{
        job := obj.(*v1alpha1.Job)
        klog.V(4).Infof("Adding job %s/%s", job.Namespace, job.Name)

        if job.DeletionTimestamp == nil &amp;&amp; needsCleanup(job) </span><span class="cov0" title="0">{
                gc.enqueue(job)
        }</span>
}

func (gc *gccontroller) updateJob(old, cur interface{}) <span class="cov0" title="0">{
        job := cur.(*v1alpha1.Job)
        klog.V(4).Infof("Updating job %s/%s", job.Namespace, job.Name)

        if job.DeletionTimestamp == nil &amp;&amp; needsCleanup(job) </span><span class="cov0" title="0">{
                gc.enqueue(job)
        }</span>
}

func (gc *gccontroller) enqueue(job *v1alpha1.Job) <span class="cov0" title="0">{
        klog.V(4).Infof("Add job %s/%s to cleanup", job.Namespace, job.Name)
        key, err := cache.MetaNamespaceKeyFunc(job)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("couldn't get key for object %#v: %v", job, err)
                return
        }</span>

        <span class="cov0" title="0">gc.queue.Add(key)</span>
}

func (gc *gccontroller) enqueueAfter(job *v1alpha1.Job, after time.Duration) <span class="cov8" title="1">{
        key, err := cache.MetaNamespaceKeyFunc(job)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("couldn't get key for object %#v: %v", job, err)
                return
        }</span>

        <span class="cov8" title="1">gc.queue.AddAfter(key, after)</span>
}

func (gc *gccontroller) worker() <span class="cov0" title="0">{
        for gc.processNextWorkItem() </span>{<span class="cov0" title="0">
        }</span>
}

func (gc *gccontroller) processNextWorkItem() bool <span class="cov0" title="0">{
        key, quit := gc.queue.Get()
        if quit </span><span class="cov0" title="0">{
                return false
        }</span>
        <span class="cov0" title="0">defer gc.queue.Done(key)

        err := gc.processJob(key.(string))
        gc.handleErr(err, key)

        return true</span>
}

func (gc *gccontroller) handleErr(err error, key interface{}) <span class="cov0" title="0">{
        if err == nil </span><span class="cov0" title="0">{
                gc.queue.Forget(key)
                return
        }</span>

        <span class="cov0" title="0">klog.Errorf("error cleaning up Job %v, will retry: %v", key, err)
        gc.queue.AddRateLimited(key)</span>
}

// processJob will check the Job's state and TTL and delete the Job when it
// finishes and its TTL after finished has expired. If the Job hasn't finished or
// its TTL hasn't expired, it will be added to the queue after the TTL is expected
// to expire.
// This function is not meant to be invoked concurrently with the same key.
func (gc *gccontroller) processJob(key string) error <span class="cov0" title="0">{
        namespace, name, err := cache.SplitMetaNamespaceKey(key)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">klog.V(4).Infof("Checking if Job %s/%s is ready for cleanup", namespace, name)
        // Ignore the Jobs that are already deleted or being deleted, or the ones that don't need clean up.
        job, err := gc.jobLister.Jobs(namespace).Get(name)
        if errors.IsNotFound(err) </span><span class="cov0" title="0">{
                return nil
        }</span>
        <span class="cov0" title="0">if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">if expired, err := gc.processTTL(job); err != nil </span><span class="cov0" title="0">{
                return err
        }</span> else<span class="cov0" title="0"> if !expired </span><span class="cov0" title="0">{
                return nil
        }</span>

        // The Job's TTL is assumed to have expired, but the Job TTL might be stale.
        // Before deleting the Job, do a final sanity check.
        // If TTL is modified before we do this check, we cannot be sure if the TTL truly expires.
        // The latest Job may have a different UID, but it's fine because the checks will be run again.
        <span class="cov0" title="0">fresh, err := gc.vcClient.BatchV1alpha1().Jobs(namespace).Get(context.TODO(), name, metav1.GetOptions{})
        if errors.IsNotFound(err) </span><span class="cov0" title="0">{
                return nil
        }</span>
        <span class="cov0" title="0">if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>
        // Use the latest Job TTL to see if the TTL truly expires.
        <span class="cov0" title="0">if expired, err := gc.processTTL(fresh); err != nil </span><span class="cov0" title="0">{
                return err
        }</span> else<span class="cov0" title="0"> if !expired </span><span class="cov0" title="0">{
                return nil
        }</span>
        // Cascade deletes the Jobs if TTL truly expires.
        <span class="cov0" title="0">policy := metav1.DeletePropagationForeground
        options := metav1.DeleteOptions{
                PropagationPolicy: &amp;policy,
                Preconditions:     &amp;metav1.Preconditions{UID: &amp;fresh.UID},
        }
        klog.V(4).Infof("Cleaning up Job %s/%s", namespace, name)
        return gc.vcClient.BatchV1alpha1().Jobs(fresh.Namespace).Delete(context.TODO(), fresh.Name, options)</span>
}

// processTTL checks whether a given Job's TTL has expired, and add it to the queue after the TTL is expected to expire
// if the TTL will expire later.
func (gc *gccontroller) processTTL(job *v1alpha1.Job) (expired bool, err error) <span class="cov8" title="1">{
        // We don't care about the Jobs that are going to be deleted, or the ones that don't need clean up.
        if job.DeletionTimestamp != nil || !needsCleanup(job) </span><span class="cov0" title="0">{
                return false, nil
        }</span>

        <span class="cov8" title="1">now := time.Now()
        t, err := timeLeft(job, &amp;now)
        if err != nil </span><span class="cov0" title="0">{
                return false, err
        }</span>

        // TTL has expired
        <span class="cov8" title="1">if *t &lt;= 0 </span><span class="cov8" title="1">{
                return true, nil
        }</span>

        <span class="cov8" title="1">gc.enqueueAfter(job, *t)
        return false, nil</span>
}

// needsCleanup checks whether a Job has finished and has a TTL set.
func needsCleanup(j *v1alpha1.Job) bool <span class="cov8" title="1">{
        return j.Spec.TTLSecondsAfterFinished != nil &amp;&amp; isJobFinished(j)
}</span>

func isJobFinished(job *v1alpha1.Job) bool <span class="cov8" title="1">{
        return job.Status.State.Phase == v1alpha1.Completed ||
                job.Status.State.Phase == v1alpha1.Failed ||
                job.Status.State.Phase == v1alpha1.Terminated
}</span>

func getFinishAndExpireTime(j *v1alpha1.Job) (*time.Time, *time.Time, error) <span class="cov8" title="1">{
        if !needsCleanup(j) </span><span class="cov8" title="1">{
                return nil, nil, fmt.Errorf("job %s/%s should not be cleaned up", j.Namespace, j.Name)
        }</span>
        <span class="cov8" title="1">finishAt, err := jobFinishTime(j)
        if err != nil </span><span class="cov0" title="0">{
                return nil, nil, err
        }</span>
        <span class="cov8" title="1">finishAtUTC := finishAt.UTC()
        expireAtUTC := finishAtUTC.Add(time.Duration(*j.Spec.TTLSecondsAfterFinished) * time.Second)
        return &amp;finishAtUTC, &amp;expireAtUTC, nil</span>
}

func timeLeft(j *v1alpha1.Job, since *time.Time) (*time.Duration, error) <span class="cov8" title="1">{
        finishAt, expireAt, err := getFinishAndExpireTime(j)
        if err != nil </span><span class="cov8" title="1">{
                return nil, err
        }</span>
        <span class="cov8" title="1">if finishAt.UTC().After(since.UTC()) </span><span class="cov0" title="0">{
                klog.Warningf("Warning: Found Job %s/%s finished in the future. This is likely due to time skew in the cluster. Job cleanup will be deferred.", j.Namespace, j.Name)
        }</span>
        <span class="cov8" title="1">remaining := expireAt.UTC().Sub(since.UTC())
        klog.V(4).Infof("Found Job %s/%s finished at %v, remaining TTL %v since %v, TTL will expire at %v", j.Namespace, j.Name, finishAt.UTC(), remaining, since.UTC(), expireAt.UTC())
        return &amp;remaining, nil</span>
}

// jobFinishTime takes an already finished Job and returns the time it finishes.
func jobFinishTime(finishedJob *v1alpha1.Job) (metav1.Time, error) <span class="cov8" title="1">{
        if finishedJob.Status.State.LastTransitionTime.IsZero() </span><span class="cov8" title="1">{
                return metav1.Time{}, fmt.Errorf("unable to find the time when the Job %s/%s finished", finishedJob.Namespace, finishedJob.Name)
        }</span>
        <span class="cov8" title="1">return finishedJob.Status.State.LastTransitionTime, nil</span>
}
</pre>
		
		<pre class="file" id="file23" style="display: none">/*
Copyright 2017 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "fmt"
        "hash"
        "hash/fnv"
        "time"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/util/wait"
        coreinformers "k8s.io/client-go/informers/core/v1"
        kubeschedulinginformers "k8s.io/client-go/informers/scheduling/v1beta1"
        "k8s.io/client-go/kubernetes"
        corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
        corelisters "k8s.io/client-go/listers/core/v1"
        kubeschedulinglisters "k8s.io/client-go/listers/scheduling/v1beta1"
        "k8s.io/client-go/tools/cache"
        "k8s.io/client-go/tools/record"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"

        batchv1alpha1 "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        vcclientset "volcano.sh/apis/pkg/client/clientset/versioned"
        vcscheme "volcano.sh/apis/pkg/client/clientset/versioned/scheme"
        informerfactory "volcano.sh/apis/pkg/client/informers/externalversions"
        batchinformer "volcano.sh/apis/pkg/client/informers/externalversions/batch/v1alpha1"
        businformer "volcano.sh/apis/pkg/client/informers/externalversions/bus/v1alpha1"
        schedulinginformers "volcano.sh/apis/pkg/client/informers/externalversions/scheduling/v1beta1"
        batchlister "volcano.sh/apis/pkg/client/listers/batch/v1alpha1"
        buslister "volcano.sh/apis/pkg/client/listers/bus/v1alpha1"
        schedulinglisters "volcano.sh/apis/pkg/client/listers/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/apis"
        jobcache "volcano.sh/volcano/pkg/controllers/cache"
        "volcano.sh/volcano/pkg/controllers/framework"
        "volcano.sh/volcano/pkg/controllers/job/state"
)

func init() <span class="cov8" title="1">{
        framework.RegisterController(&amp;jobcontroller{})
}</span>

// jobcontroller the Job jobcontroller type.
type jobcontroller struct {
        kubeClient kubernetes.Interface
        vcClient   vcclientset.Interface

        jobInformer   batchinformer.JobInformer
        podInformer   coreinformers.PodInformer
        pvcInformer   coreinformers.PersistentVolumeClaimInformer
        pgInformer    schedulinginformers.PodGroupInformer
        svcInformer   coreinformers.ServiceInformer
        cmdInformer   businformer.CommandInformer
        pcInformer    kubeschedulinginformers.PriorityClassInformer
        queueInformer schedulinginformers.QueueInformer

        // A store of jobs
        jobLister batchlister.JobLister
        jobSynced func() bool

        // A store of pods
        podLister corelisters.PodLister
        podSynced func() bool

        pvcLister corelisters.PersistentVolumeClaimLister
        pvcSynced func() bool

        // A store of podgroups
        pgLister schedulinglisters.PodGroupLister
        pgSynced func() bool

        // A store of service
        svcLister corelisters.ServiceLister
        svcSynced func() bool

        cmdLister buslister.CommandLister
        cmdSynced func() bool

        pcLister kubeschedulinglisters.PriorityClassLister
        pcSynced func() bool

        queueLister schedulinglisters.QueueLister
        queueSynced func() bool

        // queue that need to sync up
        queueList    []workqueue.RateLimitingInterface
        commandQueue workqueue.RateLimitingInterface
        cache        jobcache.Cache
        // Job Event recorder
        recorder record.EventRecorder

        errTasks      workqueue.RateLimitingInterface
        workers       uint32
        maxRequeueNum int
}

func (cc *jobcontroller) Name() string <span class="cov8" title="1">{
        return "job-controller"
}</span>

// Initialize creates the new Job job controller.
func (cc *jobcontroller) Initialize(opt *framework.ControllerOption) error <span class="cov8" title="1">{
        cc.kubeClient = opt.KubeClient
        cc.vcClient = opt.VolcanoClient

        sharedInformers := opt.SharedInformerFactory
        workers := opt.WorkerNum
        // Initialize event client
        eventBroadcaster := record.NewBroadcaster()
        eventBroadcaster.StartLogging(klog.Infof)
        eventBroadcaster.StartRecordingToSink(&amp;corev1.EventSinkImpl{Interface: cc.kubeClient.CoreV1().Events("")})
        recorder := eventBroadcaster.NewRecorder(vcscheme.Scheme, v1.EventSource{Component: "vc-controller-manager"})

        cc.queueList = make([]workqueue.RateLimitingInterface, workers)
        cc.commandQueue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
        cc.cache = jobcache.New()
        cc.errTasks = newRateLimitingQueue()
        cc.recorder = recorder
        cc.workers = workers
        cc.maxRequeueNum = opt.MaxRequeueNum
        if cc.maxRequeueNum &lt; 0 </span><span class="cov0" title="0">{
                cc.maxRequeueNum = -1
        }</span>

        <span class="cov8" title="1">var i uint32
        for i = 0; i &lt; workers; i++ </span><span class="cov8" title="1">{
                cc.queueList[i] = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
        }</span>

        <span class="cov8" title="1">cc.jobInformer = informerfactory.NewSharedInformerFactory(cc.vcClient, 0).Batch().V1alpha1().Jobs()
        cc.jobInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    cc.addJob,
                UpdateFunc: cc.updateJob,
                DeleteFunc: cc.deleteJob,
        })
        cc.jobLister = cc.jobInformer.Lister()
        cc.jobSynced = cc.jobInformer.Informer().HasSynced

        cc.cmdInformer = informerfactory.NewSharedInformerFactory(cc.vcClient, 0).Bus().V1alpha1().Commands()
        cc.cmdInformer.Informer().AddEventHandler(
                cache.FilteringResourceEventHandler{
                        FilterFunc: func(obj interface{}) bool </span><span class="cov0" title="0">{
                                switch v := obj.(type) </span>{
                                case *busv1alpha1.Command:<span class="cov0" title="0">
                                        if v.TargetObject != nil &amp;&amp;
                                                v.TargetObject.APIVersion == batchv1alpha1.SchemeGroupVersion.String() &amp;&amp;
                                                v.TargetObject.Kind == "Job" </span><span class="cov0" title="0">{
                                                return true
                                        }</span>

                                        <span class="cov0" title="0">return false</span>
                                default:<span class="cov0" title="0">
                                        return false</span>
                                }
                        },
                        Handler: cache.ResourceEventHandlerFuncs{
                                AddFunc: cc.addCommand,
                        },
                },
        )
        <span class="cov8" title="1">cc.cmdLister = cc.cmdInformer.Lister()
        cc.cmdSynced = cc.cmdInformer.Informer().HasSynced

        cc.podInformer = sharedInformers.Core().V1().Pods()
        cc.podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    cc.addPod,
                UpdateFunc: cc.updatePod,
                DeleteFunc: cc.deletePod,
        })

        cc.podLister = cc.podInformer.Lister()
        cc.podSynced = cc.podInformer.Informer().HasSynced

        cc.pvcInformer = sharedInformers.Core().V1().PersistentVolumeClaims()
        cc.pvcLister = cc.pvcInformer.Lister()
        cc.pvcSynced = cc.pvcInformer.Informer().HasSynced

        cc.svcInformer = sharedInformers.Core().V1().Services()
        cc.svcLister = cc.svcInformer.Lister()
        cc.svcSynced = cc.svcInformer.Informer().HasSynced

        cc.pgInformer = informerfactory.NewSharedInformerFactory(cc.vcClient, 0).Scheduling().V1beta1().PodGroups()
        cc.pgInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                UpdateFunc: cc.updatePodGroup,
        })
        cc.pgLister = cc.pgInformer.Lister()
        cc.pgSynced = cc.pgInformer.Informer().HasSynced

        cc.pcInformer = sharedInformers.Scheduling().V1beta1().PriorityClasses()
        cc.pcLister = cc.pcInformer.Lister()
        cc.pcSynced = cc.pcInformer.Informer().HasSynced

        cc.queueInformer = informerfactory.NewSharedInformerFactory(cc.vcClient, 0).Scheduling().V1beta1().Queues()
        cc.queueLister = cc.queueInformer.Lister()
        cc.queueSynced = cc.queueInformer.Informer().HasSynced

        // Register actions
        state.SyncJob = cc.syncJob
        state.KillJob = cc.killJob

        return nil</span>
}

// Run start JobController.
func (cc *jobcontroller) Run(stopCh &lt;-chan struct{}) <span class="cov0" title="0">{
        go cc.jobInformer.Informer().Run(stopCh)
        go cc.podInformer.Informer().Run(stopCh)
        go cc.pvcInformer.Informer().Run(stopCh)
        go cc.pgInformer.Informer().Run(stopCh)
        go cc.svcInformer.Informer().Run(stopCh)
        go cc.cmdInformer.Informer().Run(stopCh)
        go cc.pcInformer.Informer().Run(stopCh)
        go cc.queueInformer.Informer().Run(stopCh)

        cache.WaitForCacheSync(stopCh, cc.jobSynced, cc.podSynced, cc.pgSynced,
                cc.svcSynced, cc.cmdSynced, cc.pvcSynced, cc.pcSynced, cc.queueSynced)

        go wait.Until(cc.handleCommands, 0, stopCh)
        var i uint32
        for i = 0; i &lt; cc.workers; i++ </span><span class="cov0" title="0">{
                go func(num uint32) </span><span class="cov0" title="0">{
                        wait.Until(
                                func() </span><span class="cov0" title="0">{
                                        cc.worker(num)
                                }</span>,
                                time.Second,
                                stopCh)
                }(i)
        }

        <span class="cov0" title="0">go cc.cache.Run(stopCh)

        // Re-sync error tasks.
        go wait.Until(cc.processResyncTask, 0, stopCh)

        klog.Infof("JobController is running ...... ")</span>
}

func (cc *jobcontroller) worker(i uint32) <span class="cov0" title="0">{
        klog.Infof("worker %d start ...... ", i)

        for cc.processNextReq(i) </span>{<span class="cov0" title="0">
        }</span>
}

func (cc *jobcontroller) belongsToThisRoutine(key string, count uint32) bool <span class="cov0" title="0">{
        var hashVal hash.Hash32
        var val uint32

        hashVal = fnv.New32()
        hashVal.Write([]byte(key))

        val = hashVal.Sum32()

        return val%cc.workers == count
}</span>

func (cc *jobcontroller) getWorkerQueue(key string) workqueue.RateLimitingInterface <span class="cov8" title="1">{
        var hashVal hash.Hash32
        var val uint32

        hashVal = fnv.New32()
        hashVal.Write([]byte(key))

        val = hashVal.Sum32()

        queue := cc.queueList[val%cc.workers]

        return queue
}</span>

func (cc *jobcontroller) processNextReq(count uint32) bool <span class="cov0" title="0">{
        queue := cc.queueList[count]
        obj, shutdown := queue.Get()
        if shutdown </span><span class="cov0" title="0">{
                klog.Errorf("Fail to pop item from queue")
                return false
        }</span>

        <span class="cov0" title="0">req := obj.(apis.Request)
        defer queue.Done(req)

        key := jobcache.JobKeyByReq(&amp;req)
        if !cc.belongsToThisRoutine(key, count) </span><span class="cov0" title="0">{
                klog.Errorf("should not occur The job does not belongs to this routine key:%s, worker:%d...... ", key, count)
                queueLocal := cc.getWorkerQueue(key)
                queueLocal.Add(req)
                return true
        }</span>

        <span class="cov0" title="0">klog.V(3).Infof("Try to handle request &lt;%v&gt;", req)

        jobInfo, err := cc.cache.Get(key)
        if err != nil </span><span class="cov0" title="0">{
                // TODO(k82cn): ignore not-ready error.
                klog.Errorf("Failed to get job by &lt;%v&gt; from cache: %v", req, err)
                return true
        }</span>

        <span class="cov0" title="0">st := state.NewState(jobInfo)
        if st == nil </span><span class="cov0" title="0">{
                klog.Errorf("Invalid state &lt;%s&gt; of Job &lt;%v/%v&gt;",
                        jobInfo.Job.Status.State, jobInfo.Job.Namespace, jobInfo.Job.Name)
                return true
        }</span>

        <span class="cov0" title="0">action := applyPolicies(jobInfo.Job, &amp;req)
        klog.V(3).Infof("Execute &lt;%v&gt; on Job &lt;%s/%s&gt; in &lt;%s&gt; by &lt;%T&gt;.",
                action, req.Namespace, req.JobName, jobInfo.Job.Status.State.Phase, st)

        if action != busv1alpha1.SyncJobAction </span><span class="cov0" title="0">{
                cc.recordJobEvent(jobInfo.Job.Namespace, jobInfo.Job.Name, batchv1alpha1.ExecuteAction, fmt.Sprintf(
                        "Start to execute action %s ", action))
        }</span>

        <span class="cov0" title="0">if err := st.Execute(action); err != nil </span><span class="cov0" title="0">{
                if cc.maxRequeueNum == -1 || queue.NumRequeues(req) &lt; cc.maxRequeueNum </span><span class="cov0" title="0">{
                        klog.V(2).Infof("Failed to handle Job &lt;%s/%s&gt;: %v",
                                jobInfo.Job.Namespace, jobInfo.Job.Name, err)
                        // If any error, requeue it.
                        queue.AddRateLimited(req)
                        return true
                }</span>
                <span class="cov0" title="0">cc.recordJobEvent(jobInfo.Job.Namespace, jobInfo.Job.Name, batchv1alpha1.ExecuteAction, fmt.Sprintf(
                        "Job failed on action %s for retry limit reached", action))
                klog.Warningf("Terminating Job &lt;%s/%s&gt; and releasing resources", jobInfo.Job.Namespace, jobInfo.Job.Name)
                if err = st.Execute(busv1alpha1.TerminateJobAction); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to terminate Job&lt;%s/%s&gt;: %v", jobInfo.Job.Namespace, jobInfo.Job.Name, err)
                }</span>
                <span class="cov0" title="0">klog.Warningf("Dropping job&lt;%s/%s&gt; out of the queue: %v because max retries has reached", jobInfo.Job.Namespace, jobInfo.Job.Name, err)</span>
        }

        // If no error, forget it.
        <span class="cov0" title="0">queue.Forget(req)

        return true</span>
}
</pre>
		
		<pre class="file" id="file24" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"
        "reflect"
        "sort"
        "sync"
        "sync/atomic"
        "time"

        v1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/klog"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        scheduling "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/apis"
        jobhelpers "volcano.sh/volcano/pkg/controllers/job/helpers"
        "volcano.sh/volcano/pkg/controllers/job/state"
)

var calMutex sync.Mutex

func (cc *jobcontroller) killJob(jobInfo *apis.JobInfo, podRetainPhase state.PhaseMap, updateStatus state.UpdateStatusFn) error <span class="cov8" title="1">{
        job := jobInfo.Job
        klog.V(3).Infof("Killing Job &lt;%s/%s&gt;, current version %d", job.Namespace, job.Name, job.Status.Version)
        defer klog.V(3).Infof("Finished Job &lt;%s/%s&gt; killing, current version %d", job.Namespace, job.Name, job.Status.Version)

        if job.DeletionTimestamp != nil </span><span class="cov0" title="0">{
                klog.Infof("Job &lt;%s/%s&gt; is terminating, skip management process.",
                        job.Namespace, job.Name)
                return nil
        }</span>

        <span class="cov8" title="1">var pending, running, terminating, succeeded, failed, unknown int32
        taskStatusCount := make(map[string]batch.TaskState)

        var errs []error
        var total int

        for _, pods := range jobInfo.Pods </span><span class="cov8" title="1">{
                for _, pod := range pods </span><span class="cov8" title="1">{
                        total++

                        if pod.DeletionTimestamp != nil </span><span class="cov0" title="0">{
                                klog.Infof("Pod &lt;%s/%s&gt; is terminating", pod.Namespace, pod.Name)
                                terminating++
                                continue</span>
                        }

                        <span class="cov8" title="1">maxRetry := job.Spec.MaxRetry
                        lastRetry := false
                        if job.Status.RetryCount &gt; maxRetry-1 </span><span class="cov8" title="1">{
                                lastRetry = true
                        }</span>

                        <span class="cov8" title="1">_, retain := podRetainPhase[pod.Status.Phase]

                        if !retain &amp;&amp; !lastRetry </span><span class="cov0" title="0">{
                                err := cc.deleteJobPod(job.Name, pod)
                                if err == nil </span><span class="cov0" title="0">{
                                        terminating++
                                        continue</span>
                                }
                                // record the err, and then collect the pod info like retained pod
                                <span class="cov0" title="0">errs = append(errs, err)
                                cc.resyncTask(pod)</span>
                        }

                        <span class="cov8" title="1">classifyAndAddUpPodBaseOnPhase(pod, &amp;pending, &amp;running, &amp;succeeded, &amp;failed, &amp;unknown)
                        calcPodStatus(pod, taskStatusCount)</span>
                }
        }

        <span class="cov8" title="1">if len(errs) != 0 </span><span class="cov0" title="0">{
                klog.Errorf("failed to kill pods for job %s/%s, with err %+v", job.Namespace, job.Name, errs)
                cc.recorder.Event(job, v1.EventTypeWarning, FailedDeletePodReason,
                        fmt.Sprintf("Error deleting pods: %+v", errs))
                return fmt.Errorf("failed to kill %d pods of %d", len(errs), total)
        }</span>

        <span class="cov8" title="1">job = job.DeepCopy()
        // Job version is bumped only when job is killed
        job.Status.Version++
        job.Status.Pending = pending
        job.Status.Running = running
        job.Status.Succeeded = succeeded
        job.Status.Failed = failed
        job.Status.Terminating = terminating
        job.Status.Unknown = unknown
        job.Status.TaskStatusCount = taskStatusCount

        // Update running duration
        klog.V(3).Infof("Running duration is %s", metav1.Duration{Duration: time.Since(jobInfo.Job.CreationTimestamp.Time)}.ToUnstructured())
        job.Status.RunningDuration = &amp;metav1.Duration{Duration: time.Since(jobInfo.Job.CreationTimestamp.Time)}

        if updateStatus != nil </span><span class="cov8" title="1">{
                if updateStatus(&amp;job.Status) </span><span class="cov8" title="1">{
                        job.Status.State.LastTransitionTime = metav1.Now()
                }</span>
        }

        // must be called before update job status
        <span class="cov8" title="1">if err := cc.pluginOnJobDelete(job); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        // Update Job status
        <span class="cov8" title="1">newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(context.TODO(), job, metav1.UpdateOptions{})
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to update status of Job %v/%v: %v",
                        job.Namespace, job.Name, err)
                return err
        }</span>
        <span class="cov8" title="1">if e := cc.cache.Update(newJob); e != nil </span><span class="cov0" title="0">{
                klog.Errorf("KillJob - Failed to update Job %v/%v in cache:  %v",
                        newJob.Namespace, newJob.Name, e)
                return e
        }</span>

        // Delete PodGroup
        <span class="cov8" title="1">if err := cc.vcClient.SchedulingV1beta1().PodGroups(job.Namespace).Delete(context.TODO(), job.Name, metav1.DeleteOptions{}); err != nil </span><span class="cov8" title="1">{
                if !apierrors.IsNotFound(err) </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to delete PodGroup of Job %v/%v: %v",
                                job.Namespace, job.Name, err)
                        return err
                }</span>
        }

        // NOTE(k82cn): DO NOT delete input/output until job is deleted.

        <span class="cov8" title="1">return nil</span>
}

func (cc *jobcontroller) initiateJob(job *batch.Job) (*batch.Job, error) <span class="cov8" title="1">{
        klog.V(3).Infof("Starting to initiate Job &lt;%s/%s&gt;", job.Namespace, job.Name)
        jobInstance, err := cc.initJobStatus(job)
        if err != nil </span><span class="cov0" title="0">{
                cc.recorder.Event(job, v1.EventTypeWarning, string(batch.JobStatusError),
                        fmt.Sprintf("Failed to initialize job status, err: %v", err))
                return nil, err
        }</span>

        <span class="cov8" title="1">if err := cc.pluginOnJobAdd(jobInstance); err != nil </span><span class="cov0" title="0">{
                cc.recorder.Event(job, v1.EventTypeWarning, string(batch.PluginError),
                        fmt.Sprintf("Execute plugin when job add failed, err: %v", err))
                return nil, err
        }</span>

        <span class="cov8" title="1">newJob, err := cc.createJobIOIfNotExist(jobInstance)
        if err != nil </span><span class="cov0" title="0">{
                cc.recorder.Event(job, v1.EventTypeWarning, string(batch.PVCError),
                        fmt.Sprintf("Failed to create PVC, err: %v", err))
                return nil, err
        }</span>

        <span class="cov8" title="1">if err := cc.createOrUpdatePodGroup(newJob); err != nil </span><span class="cov0" title="0">{
                cc.recorder.Event(job, v1.EventTypeWarning, string(batch.PodGroupError),
                        fmt.Sprintf("Failed to create PodGroup, err: %v", err))
                return nil, err
        }</span>

        <span class="cov8" title="1">return newJob, nil</span>
}

func (cc *jobcontroller) initOnJobUpdate(job *batch.Job) error <span class="cov8" title="1">{
        klog.V(3).Infof("Starting to initiate Job &lt;%s/%s&gt; on update", job.Namespace, job.Name)

        if err := cc.pluginOnJobUpdate(job); err != nil </span><span class="cov0" title="0">{
                cc.recorder.Event(job, v1.EventTypeWarning, string(batch.PluginError),
                        fmt.Sprintf("Execute plugin when job add failed, err: %v", err))
                return err
        }</span>

        <span class="cov8" title="1">if err := cc.createOrUpdatePodGroup(job); err != nil </span><span class="cov0" title="0">{
                cc.recorder.Event(job, v1.EventTypeWarning, string(batch.PodGroupError),
                        fmt.Sprintf("Failed to create PodGroup, err: %v", err))
                return err
        }</span>

        <span class="cov8" title="1">return nil</span>
}

func (cc *jobcontroller) GetQueueInfo(queue string) (*scheduling.Queue, error) <span class="cov0" title="0">{
        queueInfo, err := cc.queueLister.Get(queue)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to get queue from queueLister, error: %s", err.Error())
        }</span>

        <span class="cov0" title="0">return queueInfo, err</span>
}

func (cc *jobcontroller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateStatusFn) error <span class="cov8" title="1">{
        job := jobInfo.Job
        klog.V(3).Infof("Starting to sync up Job &lt;%s/%s&gt;, current version %d", job.Namespace, job.Name, job.Status.Version)
        defer klog.V(3).Infof("Finished Job &lt;%s/%s&gt; sync up, current version %d", job.Namespace, job.Name, job.Status.Version)

        if jobInfo.Job.DeletionTimestamp != nil </span><span class="cov0" title="0">{
                klog.Infof("Job &lt;%s/%s&gt; is terminating, skip management process.",
                        jobInfo.Job.Namespace, jobInfo.Job.Name)
                return nil
        }</span>

        // deep copy job to prevent mutate it
        <span class="cov8" title="1">job = job.DeepCopy()

        // Find queue that job belongs to, and check if the queue has forwarding metadata
        queueInfo, err := cc.GetQueueInfo(job.Spec.Queue)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">var jobForwarding bool
        if len(queueInfo.Spec.ExtendClusters) != 0 </span><span class="cov0" title="0">{
                jobForwarding = true
                if len(job.Annotations) == 0 </span><span class="cov0" title="0">{
                        job.Annotations = make(map[string]string)
                }</span>
                <span class="cov0" title="0">job.Annotations[batch.JobForwardingKey] = "true"
                job, err = cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).Update(context.TODO(), job, metav1.UpdateOptions{})
                if err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("failed to update job: %s/%s, error: %s", job.Namespace, job.Name, err.Error())
                        return err
                }</span>
        }

        // Skip job initiation if job is already initiated
        <span class="cov8" title="1">if !isInitiated(job) </span><span class="cov8" title="1">{
                if job, err = cc.initiateJob(job); err != nil </span><span class="cov0" title="0">{
                        return err
                }</span>
        } else<span class="cov8" title="1"> {
                // TODO: optimize this call it only when scale up/down
                if err = cc.initOnJobUpdate(job); err != nil </span><span class="cov0" title="0">{
                        return err
                }</span>
        }

        <span class="cov8" title="1">if len(queueInfo.Spec.ExtendClusters) != 0 </span><span class="cov0" title="0">{
                jobForwarding = true
                job.Annotations[batch.JobForwardingKey] = "true"
                _, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).Update(context.TODO(), job, metav1.UpdateOptions{})
                if err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("failed to update job: %s/%s, error: %s", job.Namespace, job.Name, err.Error())
                        return err
                }</span>
        }

        <span class="cov8" title="1">var syncTask bool
        if pg, _ := cc.pgLister.PodGroups(job.Namespace).Get(job.Name); pg != nil </span><span class="cov8" title="1">{
                if pg.Status.Phase != "" &amp;&amp; pg.Status.Phase != scheduling.PodGroupPending </span><span class="cov8" title="1">{
                        syncTask = true
                }</span>

                <span class="cov8" title="1">for _, condition := range pg.Status.Conditions </span><span class="cov0" title="0">{
                        if condition.Type == scheduling.PodGroupUnschedulableType </span><span class="cov0" title="0">{
                                cc.recorder.Eventf(job, v1.EventTypeWarning, string(batch.PodGroupPending),
                                        fmt.Sprintf("PodGroup %s:%s unschedule,reason: %s", job.Namespace, job.Name, condition.Message))
                        }</span>
                }
        }

        <span class="cov8" title="1">if !syncTask </span><span class="cov8" title="1">{
                if updateStatus != nil </span><span class="cov8" title="1">{
                        if updateStatus(&amp;job.Status) </span><span class="cov8" title="1">{
                                job.Status.State.LastTransitionTime = metav1.Now()
                        }</span>
                }
                <span class="cov8" title="1">newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(context.TODO(), job, metav1.UpdateOptions{})
                if err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update status of Job %v/%v: %v",
                                job.Namespace, job.Name, err)
                        return err
                }</span>
                <span class="cov8" title="1">if e := cc.cache.Update(newJob); e != nil </span><span class="cov0" title="0">{
                        klog.Errorf("SyncJob - Failed to update Job %v/%v in cache:  %v",
                                newJob.Namespace, newJob.Name, e)
                        return e
                }</span>
                <span class="cov8" title="1">return nil</span>
        }

        <span class="cov8" title="1">var running, pending, terminating, succeeded, failed, unknown int32
        taskStatusCount := make(map[string]batch.TaskState)

        var podToCreate []*v1.Pod
        var podToDelete []*v1.Pod
        var creationErrs []error
        var deletionErrs []error
        appendMutex := sync.Mutex{}

        appendError := func(container *[]error, err error) </span><span class="cov0" title="0">{
                appendMutex.Lock()
                defer appendMutex.Unlock()
                *container = append(*container, err)
        }</span>

        <span class="cov8" title="1">for _, ts := range job.Spec.Tasks </span><span class="cov8" title="1">{
                ts.Template.Name = ts.Name
                tc := ts.Template.DeepCopy()
                name := ts.Template.Name

                pods, found := jobInfo.Pods[name]
                if !found </span><span class="cov0" title="0">{
                        pods = map[string]*v1.Pod{}
                }</span>

                <span class="cov8" title="1">for i := 0; i &lt; int(ts.Replicas); i++ </span><span class="cov8" title="1">{
                        podName := fmt.Sprintf(jobhelpers.PodNameFmt, job.Name, name, i)
                        if pod, found := pods[podName]; !found </span><span class="cov8" title="1">{
                                newPod := createJobPod(job, tc, ts.TopologyPolicy, i, jobForwarding)
                                if err := cc.pluginOnPodCreate(job, newPod); err != nil </span><span class="cov0" title="0">{
                                        return err
                                }</span>
                                <span class="cov8" title="1">podToCreate = append(podToCreate, newPod)</span>
                        } else<span class="cov8" title="1"> {
                                delete(pods, podName)
                                if pod.DeletionTimestamp != nil </span><span class="cov0" title="0">{
                                        klog.Infof("Pod &lt;%s/%s&gt; is terminating", pod.Namespace, pod.Name)
                                        atomic.AddInt32(&amp;terminating, 1)
                                        continue</span>
                                }

                                <span class="cov8" title="1">classifyAndAddUpPodBaseOnPhase(pod, &amp;pending, &amp;running, &amp;succeeded, &amp;failed, &amp;unknown)
                                calcPodStatus(pod, taskStatusCount)</span>
                        }
                }

                <span class="cov8" title="1">for _, pod := range pods </span><span class="cov0" title="0">{
                        podToDelete = append(podToDelete, pod)
                }</span>
        }

        <span class="cov8" title="1">waitCreationGroup := sync.WaitGroup{}
        waitCreationGroup.Add(len(podToCreate))
        for _, pod := range podToCreate </span><span class="cov8" title="1">{
                go func(pod *v1.Pod) </span><span class="cov8" title="1">{
                        defer waitCreationGroup.Done()
                        newPod, err := cc.kubeClient.CoreV1().Pods(pod.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{})
                        if err != nil &amp;&amp; !apierrors.IsAlreadyExists(err) </span><span class="cov0" title="0">{
                                // Failed to create Pod, waitCreationGroup a moment and then create it again
                                // This is to ensure all podsMap under the same Job created
                                // So gang-scheduling could schedule the Job successfully
                                klog.Errorf("Failed to create pod %s for Job %s, err %#v",
                                        pod.Name, job.Name, err)
                                appendError(&amp;creationErrs, fmt.Errorf("failed to create pod %s, err: %#v", pod.Name, err))
                        }</span> else<span class="cov8" title="1"> {
                                classifyAndAddUpPodBaseOnPhase(newPod, &amp;pending, &amp;running, &amp;succeeded, &amp;failed, &amp;unknown)
                                calcPodStatus(pod, taskStatusCount)
                                klog.V(3).Infof("Created Task &lt;%s&gt; of Job &lt;%s/%s&gt;",
                                        pod.Name, job.Namespace, job.Name)
                        }</span>
                }(pod)
        }
        <span class="cov8" title="1">waitCreationGroup.Wait()

        if len(creationErrs) != 0 </span><span class="cov0" title="0">{
                cc.recorder.Event(job, v1.EventTypeWarning, FailedCreatePodReason,
                        fmt.Sprintf("Error creating pods: %+v", creationErrs))
                return fmt.Errorf("failed to create %d pods of %d", len(creationErrs), len(podToCreate))
        }</span>

        // Delete pods when scale down.
        <span class="cov8" title="1">waitDeletionGroup := sync.WaitGroup{}
        waitDeletionGroup.Add(len(podToDelete))
        for _, pod := range podToDelete </span><span class="cov0" title="0">{
                go func(pod *v1.Pod) </span><span class="cov0" title="0">{
                        defer waitDeletionGroup.Done()
                        err := cc.deleteJobPod(job.Name, pod)
                        if err != nil </span><span class="cov0" title="0">{
                                // Failed to delete Pod, waitCreationGroup a moment and then create it again
                                // This is to ensure all podsMap under the same Job created
                                // So gang-scheduling could schedule the Job successfully
                                klog.Errorf("Failed to delete pod %s for Job %s, err %#v",
                                        pod.Name, job.Name, err)
                                appendError(&amp;deletionErrs, err)
                                cc.resyncTask(pod)
                        }</span> else<span class="cov0" title="0"> {
                                klog.V(3).Infof("Deleted Task &lt;%s&gt; of Job &lt;%s/%s&gt;",
                                        pod.Name, job.Namespace, job.Name)
                                atomic.AddInt32(&amp;terminating, 1)
                        }</span>
                }(pod)
        }
        <span class="cov8" title="1">waitDeletionGroup.Wait()

        if len(deletionErrs) != 0 </span><span class="cov0" title="0">{
                cc.recorder.Event(job, v1.EventTypeWarning, FailedDeletePodReason,
                        fmt.Sprintf("Error deleting pods: %+v", deletionErrs))
                return fmt.Errorf("failed to delete %d pods of %d", len(deletionErrs), len(podToDelete))
        }</span>
        <span class="cov8" title="1">job.Status = batch.JobStatus{
                State: job.Status.State,

                Pending:             pending,
                Running:             running,
                Succeeded:           succeeded,
                Failed:              failed,
                Terminating:         terminating,
                Unknown:             unknown,
                Version:             job.Status.Version,
                MinAvailable:        job.Spec.MinAvailable,
                TaskStatusCount:     taskStatusCount,
                ControlledResources: job.Status.ControlledResources,
                RetryCount:          job.Status.RetryCount,
        }

        if updateStatus != nil </span><span class="cov0" title="0">{
                if updateStatus(&amp;job.Status) </span><span class="cov0" title="0">{
                        job.Status.State.LastTransitionTime = metav1.Now()
                }</span>
        }
        <span class="cov8" title="1">newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(context.TODO(), job, metav1.UpdateOptions{})
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to update status of Job %v/%v: %v",
                        job.Namespace, job.Name, err)
                return err
        }</span>
        <span class="cov8" title="1">if e := cc.cache.Update(newJob); e != nil </span><span class="cov0" title="0">{
                klog.Errorf("SyncJob - Failed to update Job %v/%v in cache:  %v",
                        newJob.Namespace, newJob.Name, e)
                return e
        }</span>

        <span class="cov8" title="1">return nil</span>
}

func (cc *jobcontroller) createJobIOIfNotExist(job *batch.Job) (*batch.Job, error) <span class="cov8" title="1">{
        // If PVC does not exist, create them for Job.
        var needUpdate bool
        if job.Status.ControlledResources == nil </span><span class="cov8" title="1">{
                job.Status.ControlledResources = make(map[string]string)
        }</span>
        <span class="cov8" title="1">for index, volume := range job.Spec.Volumes </span><span class="cov8" title="1">{
                vcName := volume.VolumeClaimName
                if len(vcName) == 0 </span><span class="cov0" title="0">{
                        // NOTE(k82cn): Ensure never have duplicated generated names.
                        for </span><span class="cov0" title="0">{
                                vcName = jobhelpers.GenPVCName(job.Name)
                                exist, err := cc.checkPVCExist(job, vcName)
                                if err != nil </span><span class="cov0" title="0">{
                                        return job, err
                                }</span>
                                <span class="cov0" title="0">if exist </span><span class="cov0" title="0">{
                                        continue</span>
                                }
                                <span class="cov0" title="0">job.Spec.Volumes[index].VolumeClaimName = vcName
                                needUpdate = true
                                break</span>
                        }
                        // TODO: check VolumeClaim must be set if VolumeClaimName is empty
                        <span class="cov0" title="0">if volume.VolumeClaim != nil </span><span class="cov0" title="0">{
                                if err := cc.createPVC(job, vcName, volume.VolumeClaim); err != nil </span><span class="cov0" title="0">{
                                        return job, err
                                }</span>
                        }
                } else<span class="cov8" title="1"> {
                        exist, err := cc.checkPVCExist(job, vcName)
                        if err != nil </span><span class="cov0" title="0">{
                                return job, err
                        }</span>
                        <span class="cov8" title="1">if !exist </span><span class="cov8" title="1">{
                                return job, fmt.Errorf("pvc %s is not found, the job will be in the Pending state until the PVC is created", vcName)
                        }</span>
                }
                <span class="cov0" title="0">job.Status.ControlledResources["volume-pvc-"+vcName] = vcName</span>
        }
        <span class="cov8" title="1">if needUpdate </span><span class="cov0" title="0">{
                newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).Update(context.TODO(), job, metav1.UpdateOptions{})
                if err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update Job %v/%v for volume claim name: %v ",
                                job.Namespace, job.Name, err)
                        return job, err
                }</span>

                <span class="cov0" title="0">newJob.Status = job.Status
                return newJob, err</span>
        }
        <span class="cov8" title="1">return job, nil</span>
}

func (cc *jobcontroller) checkPVCExist(job *batch.Job, pvc string) (bool, error) <span class="cov8" title="1">{
        if _, err := cc.pvcLister.PersistentVolumeClaims(job.Namespace).Get(pvc); err != nil </span><span class="cov8" title="1">{
                if apierrors.IsNotFound(err) </span><span class="cov8" title="1">{
                        return false, nil
                }</span>
                <span class="cov0" title="0">klog.V(3).Infof("Failed to get PVC %s for job &lt;%s/%s&gt;: %v",
                        pvc, job.Namespace, job.Name, err)
                return false, err</span>
        }
        <span class="cov0" title="0">return true, nil</span>
}

func (cc *jobcontroller) createPVC(job *batch.Job, vcName string, volumeClaim *v1.PersistentVolumeClaimSpec) error <span class="cov8" title="1">{
        pvc := &amp;v1.PersistentVolumeClaim{
                ObjectMeta: metav1.ObjectMeta{
                        Namespace: job.Namespace,
                        Name:      vcName,
                        OwnerReferences: []metav1.OwnerReference{
                                *metav1.NewControllerRef(job, helpers.JobKind),
                        },
                },
                Spec: *volumeClaim,
        }

        klog.V(3).Infof("Try to create PVC: %v", pvc)

        if _, e := cc.kubeClient.CoreV1().PersistentVolumeClaims(job.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{}); e != nil </span><span class="cov0" title="0">{
                klog.V(3).Infof("Failed to create PVC for Job &lt;%s/%s&gt;: %v",
                        job.Namespace, job.Name, e)
                return e
        }</span>
        <span class="cov8" title="1">return nil</span>
}

func (cc *jobcontroller) createOrUpdatePodGroup(job *batch.Job) error <span class="cov8" title="1">{
        // If PodGroup does not exist, create one for Job.
        pg, err := cc.pgLister.PodGroups(job.Namespace).Get(job.Name)
        if err != nil </span><span class="cov8" title="1">{
                if !apierrors.IsNotFound(err) </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to get PodGroup for Job &lt;%s/%s&gt;: %v",
                                job.Namespace, job.Name, err)
                        return err
                }</span>

                <span class="cov8" title="1">minTaskMember := map[string]int32{}
                for _, task := range job.Spec.Tasks </span><span class="cov8" title="1">{
                        if task.MinAvailable != nil </span><span class="cov0" title="0">{
                                minTaskMember[task.Name] = *task.MinAvailable
                        }</span> else<span class="cov8" title="1"> {
                                minTaskMember[task.Name] = task.Replicas
                        }</span>
                }

                <span class="cov8" title="1">pg := &amp;scheduling.PodGroup{
                        ObjectMeta: metav1.ObjectMeta{
                                Namespace:   job.Namespace,
                                Name:        job.Name,
                                Annotations: job.Annotations,
                                Labels:      job.Labels,
                                OwnerReferences: []metav1.OwnerReference{
                                        *metav1.NewControllerRef(job, helpers.JobKind),
                                },
                        },
                        Spec: scheduling.PodGroupSpec{
                                MinMember:         job.Spec.MinAvailable,
                                MinTaskMember:     minTaskMember,
                                Queue:             job.Spec.Queue,
                                MinResources:      cc.calcPGMinResources(job),
                                PriorityClassName: job.Spec.PriorityClassName,
                        },
                }

                if _, err = cc.vcClient.SchedulingV1beta1().PodGroups(job.Namespace).Create(context.TODO(), pg, metav1.CreateOptions{}); err != nil </span><span class="cov0" title="0">{
                        if !apierrors.IsAlreadyExists(err) </span><span class="cov0" title="0">{
                                klog.Errorf("Failed to create PodGroup for Job &lt;%s/%s&gt;: %v",
                                        job.Namespace, job.Name, err)
                                return err
                        }</span>
                }
                <span class="cov8" title="1">return nil</span>
        }

        <span class="cov8" title="1">pgShouldUpdate := false
        if pg.Spec.PriorityClassName != job.Spec.PriorityClassName </span><span class="cov8" title="1">{
                pg.Spec.PriorityClassName = job.Spec.PriorityClassName
                pgShouldUpdate = true
        }</span>

        <span class="cov8" title="1">minResources := cc.calcPGMinResources(job)
        if pg.Spec.MinMember != job.Spec.MinAvailable || !reflect.DeepEqual(pg.Spec.MinResources, minResources) </span><span class="cov0" title="0">{
                pg.Spec.MinMember = job.Spec.MinAvailable
                pg.Spec.MinResources = minResources
                pgShouldUpdate = true
        }</span>

        <span class="cov8" title="1">if pg.Spec.MinTaskMember == nil </span><span class="cov8" title="1">{
                pgShouldUpdate = true
                pg.Spec.MinTaskMember = make(map[string]int32)
        }</span>

        <span class="cov8" title="1">for _, task := range job.Spec.Tasks </span><span class="cov8" title="1">{
                if task.MinAvailable == nil </span><span class="cov8" title="1">{
                        continue</span>
                }

                <span class="cov0" title="0">if taskMember, ok := pg.Spec.MinTaskMember[task.Name]; !ok </span><span class="cov0" title="0">{
                        pgShouldUpdate = true
                        pg.Spec.MinTaskMember[task.Name] = *task.MinAvailable
                }</span> else<span class="cov0" title="0"> {
                        if taskMember == *task.MinAvailable </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov0" title="0">pgShouldUpdate = true
                        pg.Spec.MinTaskMember[task.Name] = *task.MinAvailable</span>
                }
        }

        <span class="cov8" title="1">if !pgShouldUpdate </span><span class="cov8" title="1">{
                return nil
        }</span>

        <span class="cov8" title="1">_, err = cc.vcClient.SchedulingV1beta1().PodGroups(job.Namespace).Update(context.TODO(), pg, metav1.UpdateOptions{})
        if err != nil </span><span class="cov0" title="0">{
                klog.V(3).Infof("Failed to update PodGroup for Job &lt;%s/%s&gt;: %v",
                        job.Namespace, job.Name, err)
        }</span>
        <span class="cov8" title="1">return err</span>
}

func (cc *jobcontroller) deleteJobPod(jobName string, pod *v1.Pod) error <span class="cov8" title="1">{
        err := cc.kubeClient.CoreV1().Pods(pod.Namespace).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
        if err != nil &amp;&amp; !apierrors.IsNotFound(err) </span><span class="cov0" title="0">{
                klog.Errorf("Failed to delete pod %s/%s for Job %s, err %#v",
                        pod.Namespace, pod.Name, jobName, err)

                return fmt.Errorf("failed to delete pod %s, err %#v", pod.Name, err)
        }</span>

        <span class="cov8" title="1">return nil</span>
}

func (cc *jobcontroller) calcPGMinResources(job *batch.Job) *v1.ResourceList <span class="cov8" title="1">{
        // sort task by priorityClasses
        var tasksPriority TasksPriority
        for _, task := range job.Spec.Tasks </span><span class="cov8" title="1">{
                tp := TaskPriority{0, task}
                pc := task.Template.Spec.PriorityClassName

                priorityClass, err := cc.pcLister.Get(pc)
                if err != nil || priorityClass == nil </span><span class="cov8" title="1">{
                        klog.Warningf("Ignore task %s priority class %s: %v", task.Name, pc, err)
                }</span> else<span class="cov0" title="0"> {
                        tp.priority = priorityClass.Value
                }</span>

                <span class="cov8" title="1">tasksPriority = append(tasksPriority, tp)</span>
        }

        <span class="cov8" title="1">sort.Sort(tasksPriority)

        minAvailableTasksRes := v1.ResourceList{}
        podCnt := int32(0)
        for _, task := range tasksPriority </span><span class="cov8" title="1">{
                for i := int32(0); i &lt; task.Replicas; i++ </span><span class="cov8" title="1">{
                        if podCnt &gt;= job.Spec.MinAvailable </span><span class="cov8" title="1">{
                                break</span>
                        }
                        <span class="cov0" title="0">podCnt++
                        for _, c := range task.Template.Spec.Containers </span><span class="cov0" title="0">{
                                addResourceList(minAvailableTasksRes, c.Resources.Requests, c.Resources.Limits)
                        }</span>
                }
        }

        <span class="cov8" title="1">return &amp;minAvailableTasksRes</span>
}

func (cc *jobcontroller) initJobStatus(job *batch.Job) (*batch.Job, error) <span class="cov8" title="1">{
        if job.Status.State.Phase != "" </span><span class="cov8" title="1">{
                return job, nil
        }</span>

        <span class="cov0" title="0">job.Status.State.LastTransitionTime = metav1.Now()
        job.Status.State.Phase = batch.Pending
        job.Status.State.LastTransitionTime = metav1.Now()
        job.Status.MinAvailable = job.Spec.MinAvailable
        newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(context.TODO(), job, metav1.UpdateOptions{})
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to update status of Job %v/%v: %v",
                        job.Namespace, job.Name, err)
                return nil, err
        }</span>
        <span class="cov0" title="0">if err := cc.cache.Update(newJob); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("CreateJob - Failed to update Job %v/%v in cache:  %v",
                        newJob.Namespace, newJob.Name, err)
                return nil, err
        }</span>

        <span class="cov0" title="0">return newJob, nil</span>
}

func classifyAndAddUpPodBaseOnPhase(pod *v1.Pod, pending, running, succeeded, failed, unknown *int32) <span class="cov8" title="1">{
        switch pod.Status.Phase </span>{
        case v1.PodPending:<span class="cov8" title="1">
                atomic.AddInt32(pending, 1)</span>
        case v1.PodRunning:<span class="cov8" title="1">
                atomic.AddInt32(running, 1)</span>
        case v1.PodSucceeded:<span class="cov0" title="0">
                atomic.AddInt32(succeeded, 1)</span>
        case v1.PodFailed:<span class="cov0" title="0">
                atomic.AddInt32(failed, 1)</span>
        default:<span class="cov8" title="1">
                atomic.AddInt32(unknown, 1)</span>
        }
}

func calcPodStatus(pod *v1.Pod, taskStatusCount map[string]batch.TaskState) <span class="cov8" title="1">{
        taskName, found := pod.Annotations[batch.TaskSpecKey]
        if !found </span><span class="cov8" title="1">{
                return
        }</span>

        <span class="cov8" title="1">calMutex.Lock()
        defer calMutex.Unlock()
        if _, ok := taskStatusCount[taskName]; !ok </span><span class="cov8" title="1">{
                taskStatusCount[taskName] = batch.TaskState{
                        Phase: make(map[v1.PodPhase]int32),
                }
        }</span>

        <span class="cov8" title="1">switch pod.Status.Phase </span>{
        case v1.PodPending:<span class="cov0" title="0">
                taskStatusCount[taskName].Phase[v1.PodPending]++</span>
        case v1.PodRunning:<span class="cov0" title="0">
                taskStatusCount[taskName].Phase[v1.PodRunning]++</span>
        case v1.PodSucceeded:<span class="cov0" title="0">
                taskStatusCount[taskName].Phase[v1.PodSucceeded]++</span>
        case v1.PodFailed:<span class="cov0" title="0">
                taskStatusCount[taskName].Phase[v1.PodFailed]++</span>
        default:<span class="cov8" title="1">
                taskStatusCount[taskName].Phase[v1.PodUnknown]++</span>
        }
}

func isInitiated(job *batch.Job) bool <span class="cov8" title="1">{
        if job.Status.State.Phase == "" || job.Status.State.Phase == batch.Pending </span><span class="cov8" title="1">{
                return false
        }</span>

        <span class="cov8" title="1">return true</span>
}
</pre>
		
		<pre class="file" id="file25" style="display: none">/*
Copyright 2017 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"
        "reflect"
        "strconv"

        v1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/client-go/tools/cache"
        "k8s.io/klog"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        bus "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        scheduling "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/apis"
        jobcache "volcano.sh/volcano/pkg/controllers/cache"
        jobhelpers "volcano.sh/volcano/pkg/controllers/job/helpers"
)

func (cc *jobcontroller) addCommand(obj interface{}) <span class="cov8" title="1">{
        cmd, ok := obj.(*bus.Command)
        if !ok </span><span class="cov8" title="1">{
                klog.Errorf("obj is not Command")
                return
        }</span>

        <span class="cov8" title="1">cc.commandQueue.Add(cmd)</span>
}

func (cc *jobcontroller) addJob(obj interface{}) <span class="cov8" title="1">{
        job, ok := obj.(*batch.Job)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("obj is not Job")
                return
        }</span>

        <span class="cov8" title="1">req := apis.Request{
                Namespace: job.Namespace,
                JobName:   job.Name,

                Event: bus.OutOfSyncEvent,
        }

        // TODO(k82cn): if failed to add job, the cache should be refresh
        if err := cc.cache.Add(job); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to add job &lt;%s/%s&gt;: %v in cache",
                        job.Namespace, job.Name, err)
        }</span>
        <span class="cov8" title="1">key := jobhelpers.GetJobKeyByReq(&amp;req)
        queue := cc.getWorkerQueue(key)
        queue.Add(req)</span>
}

func (cc *jobcontroller) updateJob(oldObj, newObj interface{}) <span class="cov8" title="1">{
        newJob, ok := newObj.(*batch.Job)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("newObj is not Job")
                return
        }</span>

        <span class="cov8" title="1">oldJob, ok := oldObj.(*batch.Job)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("oldJob is not Job")
                return
        }</span>

        // No need to update if ResourceVersion is not changed
        <span class="cov8" title="1">if newJob.ResourceVersion == oldJob.ResourceVersion </span><span class="cov8" title="1">{
                klog.V(6).Infof("No need to update because job is not modified.")
                return
        }</span>

        <span class="cov8" title="1">if err := cc.cache.Update(newJob); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("UpdateJob - Failed to update job &lt;%s/%s&gt;: %v in cache",
                        newJob.Namespace, newJob.Name, err)
        }</span>

        // NOTE: Since we only reconcile job based on Spec, we will ignore other attributes
        // For Job status, it's used internally and always been updated via our controller.
        <span class="cov8" title="1">if reflect.DeepEqual(newJob.Spec, oldJob.Spec) &amp;&amp; newJob.Status.State.Phase == oldJob.Status.State.Phase </span><span class="cov0" title="0">{
                klog.V(6).Infof("Job update event is ignored since no update in 'Spec'.")
                return
        }</span>

        <span class="cov8" title="1">req := apis.Request{
                Namespace: newJob.Namespace,
                JobName:   newJob.Name,
                Event:     bus.OutOfSyncEvent,
        }
        key := jobhelpers.GetJobKeyByReq(&amp;req)
        queue := cc.getWorkerQueue(key)
        queue.Add(req)</span>
}

func (cc *jobcontroller) deleteJob(obj interface{}) <span class="cov0" title="0">{
        job, ok := obj.(*batch.Job)
        if !ok </span><span class="cov0" title="0">{
                // If we reached here it means the Job was deleted but its final state is unrecorded.
                tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Couldn't get object from tombstone %#v", obj)
                        return
                }</span>
                <span class="cov0" title="0">job, ok = tombstone.Obj.(*batch.Job)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Tombstone contained object that is not a volcano Job: %#v", obj)
                        return
                }</span>
        }

        <span class="cov0" title="0">if err := cc.cache.Delete(job); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to delete job &lt;%s/%s&gt;: %v in cache",
                        job.Namespace, job.Name, err)
        }</span>
}

func (cc *jobcontroller) addPod(obj interface{}) <span class="cov8" title="1">{
        pod, ok := obj.(*v1.Pod)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Failed to convert %v to v1.Pod", obj)
                return
        }</span>
        // Filter out pods that are not created from volcano job
        <span class="cov8" title="1">if !isControlledBy(pod, helpers.JobKind) </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov8" title="1">jobName, found := pod.Annotations[batch.JobNameKey]
        if !found </span><span class="cov0" title="0">{
                klog.Infof("Failed to find jobName of Pod &lt;%s/%s&gt;, skipping",
                        pod.Namespace, pod.Name)
                return
        }</span>

        <span class="cov8" title="1">version, found := pod.Annotations[batch.JobVersion]
        if !found </span><span class="cov0" title="0">{
                klog.Infof("Failed to find jobVersion of Pod &lt;%s/%s&gt;, skipping",
                        pod.Namespace, pod.Name)
                return
        }</span>

        <span class="cov8" title="1">dVersion, err := strconv.Atoi(version)
        if err != nil </span><span class="cov0" title="0">{
                klog.Infof("Failed to convert jobVersion of Pod &lt;%s/%s&gt; into number, skipping",
                        pod.Namespace, pod.Name)
                return
        }</span>

        <span class="cov8" title="1">if pod.DeletionTimestamp != nil </span><span class="cov0" title="0">{
                cc.deletePod(pod)
                return
        }</span>

        <span class="cov8" title="1">req := apis.Request{
                Namespace: pod.Namespace,
                JobName:   jobName,

                Event:      bus.OutOfSyncEvent,
                JobVersion: int32(dVersion),
        }

        if err := cc.cache.AddPod(pod); err != nil </span><span class="cov8" title="1">{
                klog.Errorf("Failed to add Pod &lt;%s/%s&gt;: %v to cache",
                        pod.Namespace, pod.Name, err)
        }</span>
        <span class="cov8" title="1">key := jobhelpers.GetJobKeyByReq(&amp;req)
        queue := cc.getWorkerQueue(key)
        queue.Add(req)</span>
}

func (cc *jobcontroller) updatePod(oldObj, newObj interface{}) <span class="cov8" title="1">{
        oldPod, ok := oldObj.(*v1.Pod)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Failed to convert %v to v1.Pod", oldObj)
                return
        }</span>

        <span class="cov8" title="1">newPod, ok := newObj.(*v1.Pod)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Failed to convert %v to v1.Pod", newObj)
                return
        }</span>

        // Filter out pods that are not created from volcano job
        <span class="cov8" title="1">if !isControlledBy(newPod, helpers.JobKind) </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov8" title="1">if newPod.ResourceVersion == oldPod.ResourceVersion </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov8" title="1">if newPod.DeletionTimestamp != nil </span><span class="cov0" title="0">{
                cc.deletePod(newObj)
                return
        }</span>

        <span class="cov8" title="1">taskName, found := newPod.Annotations[batch.TaskSpecKey]
        if !found </span><span class="cov0" title="0">{
                klog.Infof("Failed to find taskName of Pod &lt;%s/%s&gt;, skipping",
                        newPod.Namespace, newPod.Name)
                return
        }</span>

        <span class="cov8" title="1">jobName, found := newPod.Annotations[batch.JobNameKey]
        if !found </span><span class="cov0" title="0">{
                klog.Infof("Failed to find jobName of Pod &lt;%s/%s&gt;, skipping",
                        newPod.Namespace, newPod.Name)
                return
        }</span>

        <span class="cov8" title="1">version, found := newPod.Annotations[batch.JobVersion]
        if !found </span><span class="cov0" title="0">{
                klog.Infof("Failed to find jobVersion of Pod &lt;%s/%s&gt;, skipping",
                        newPod.Namespace, newPod.Name)
                return
        }</span>

        <span class="cov8" title="1">dVersion, err := strconv.Atoi(version)
        if err != nil </span><span class="cov0" title="0">{
                klog.Infof("Failed to convert jobVersion of Pod into number &lt;%s/%s&gt;, skipping",
                        newPod.Namespace, newPod.Name)
                return
        }</span>

        <span class="cov8" title="1">if err := cc.cache.UpdatePod(newPod); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to update Pod &lt;%s/%s&gt;: %v in cache",
                        newPod.Namespace, newPod.Name, err)
        }</span>

        <span class="cov8" title="1">event := bus.OutOfSyncEvent
        var exitCode int32

        switch newPod.Status.Phase </span>{
        case v1.PodFailed:<span class="cov8" title="1">
                if oldPod.Status.Phase != v1.PodFailed </span><span class="cov8" title="1">{
                        event = bus.PodFailedEvent
                        // TODO: currently only one container pod is supported by volcano
                        // Once multi containers pod is supported, update accordingly.
                        if len(newPod.Status.ContainerStatuses) &gt; 0 &amp;&amp; newPod.Status.ContainerStatuses[0].State.Terminated != nil </span><span class="cov0" title="0">{
                                exitCode = newPod.Status.ContainerStatuses[0].State.Terminated.ExitCode
                        }</span>
                }
        case v1.PodSucceeded:<span class="cov0" title="0">
                if oldPod.Status.Phase != v1.PodSucceeded &amp;&amp;
                        cc.cache.TaskCompleted(jobcache.JobKeyByName(newPod.Namespace, jobName), taskName) </span><span class="cov0" title="0">{
                        event = bus.TaskCompletedEvent
                }</span>
        case v1.PodPending, v1.PodRunning:<span class="cov8" title="1">
                if cc.cache.TaskFailed(jobcache.JobKeyByName(newPod.Namespace, jobName), taskName) </span><span class="cov0" title="0">{
                        event = bus.TaskFailedEvent
                }</span>
        }

        <span class="cov8" title="1">req := apis.Request{
                Namespace: newPod.Namespace,
                JobName:   jobName,
                TaskName:  taskName,

                Event:      event,
                ExitCode:   exitCode,
                JobVersion: int32(dVersion),
        }

        key := jobhelpers.GetJobKeyByReq(&amp;req)
        queue := cc.getWorkerQueue(key)
        queue.Add(req)</span>
}

func (cc *jobcontroller) deletePod(obj interface{}) <span class="cov8" title="1">{
        pod, ok := obj.(*v1.Pod)
        if !ok </span><span class="cov0" title="0">{
                // If we reached here it means the pod was deleted but its final state is unrecorded.
                tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Couldn't get object from tombstone %#v", obj)
                        return
                }</span>
                <span class="cov0" title="0">pod, ok = tombstone.Obj.(*v1.Pod)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Tombstone contained object that is not a Pod: %#v", obj)
                        return
                }</span>
        }

        // Filter out pods that are not created from volcano job
        <span class="cov8" title="1">if !isControlledBy(pod, helpers.JobKind) </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov8" title="1">taskName, found := pod.Annotations[batch.TaskSpecKey]
        if !found </span><span class="cov0" title="0">{
                klog.Infof("Failed to find taskName of Pod &lt;%s/%s&gt;, skipping",
                        pod.Namespace, pod.Name)
                return
        }</span>

        <span class="cov8" title="1">jobName, found := pod.Annotations[batch.JobNameKey]
        if !found </span><span class="cov0" title="0">{
                klog.Infof("Failed to find jobName of Pod &lt;%s/%s&gt;, skipping",
                        pod.Namespace, pod.Name)
                return
        }</span>

        <span class="cov8" title="1">version, found := pod.Annotations[batch.JobVersion]
        if !found </span><span class="cov0" title="0">{
                klog.Infof("Failed to find jobVersion of Pod &lt;%s/%s&gt;, skipping",
                        pod.Namespace, pod.Name)
                return
        }</span>

        <span class="cov8" title="1">dVersion, err := strconv.Atoi(version)
        if err != nil </span><span class="cov0" title="0">{
                klog.Infof("Failed to convert jobVersion of Pod &lt;%s/%s&gt; into number, skipping",
                        pod.Namespace, pod.Name)
                return
        }</span>

        <span class="cov8" title="1">req := apis.Request{
                Namespace: pod.Namespace,
                JobName:   jobName,
                TaskName:  taskName,

                Event:      bus.PodEvictedEvent,
                JobVersion: int32(dVersion),
        }

        if err := cc.cache.DeletePod(pod); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to delete Pod &lt;%s/%s&gt;: %v in cache",
                        pod.Namespace, pod.Name, err)
        }</span>

        <span class="cov8" title="1">key := jobhelpers.GetJobKeyByReq(&amp;req)
        queue := cc.getWorkerQueue(key)
        queue.Add(req)</span>
}

func (cc *jobcontroller) recordJobEvent(namespace, name string, event batch.JobEvent, message string) <span class="cov0" title="0">{
        job, err := cc.cache.Get(jobcache.JobKeyByName(namespace, name))
        if err != nil </span><span class="cov0" title="0">{
                klog.Warningf("Failed to find job in cache when reporting job event &lt;%s/%s&gt;: %v",
                        namespace, name, err)
                return
        }</span>
        <span class="cov0" title="0">cc.recorder.Event(job.Job, v1.EventTypeNormal, string(event), message)</span>
}

func (cc *jobcontroller) handleCommands() <span class="cov0" title="0">{
        for cc.processNextCommand() </span>{<span class="cov0" title="0">
        }</span>
}

func (cc *jobcontroller) processNextCommand() bool <span class="cov0" title="0">{
        obj, shutdown := cc.commandQueue.Get()
        if shutdown </span><span class="cov0" title="0">{
                return false
        }</span>
        <span class="cov0" title="0">cmd := obj.(*bus.Command)
        defer cc.commandQueue.Done(cmd)

        if err := cc.vcClient.BusV1alpha1().Commands(cmd.Namespace).Delete(context.TODO(), cmd.Name, metav1.DeleteOptions{}); err != nil </span><span class="cov0" title="0">{
                if !apierrors.IsNotFound(err) </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to delete Command &lt;%s/%s&gt;.", cmd.Namespace, cmd.Name)
                        cc.commandQueue.AddRateLimited(cmd)
                }</span>
                <span class="cov0" title="0">return true</span>
        }
        <span class="cov0" title="0">cc.recordJobEvent(cmd.Namespace, cmd.TargetObject.Name,
                batch.CommandIssued,
                fmt.Sprintf(
                        "Start to execute command %s, and clean it up to make sure executed not more than once.", cmd.Action))
        req := apis.Request{
                Namespace: cmd.Namespace,
                JobName:   cmd.TargetObject.Name,
                Event:     bus.CommandIssuedEvent,
                Action:    bus.Action(cmd.Action),
        }

        key := jobhelpers.GetJobKeyByReq(&amp;req)
        queue := cc.getWorkerQueue(key)
        queue.Add(req)

        return true</span>
}

func (cc *jobcontroller) updatePodGroup(oldObj, newObj interface{}) <span class="cov8" title="1">{
        oldPG, ok := oldObj.(*scheduling.PodGroup)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Failed to convert %v to PodGroup", newObj)
                return
        }</span>

        <span class="cov8" title="1">newPG, ok := newObj.(*scheduling.PodGroup)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Failed to convert %v to PodGroup", newObj)
                return
        }</span>

        <span class="cov8" title="1">_, err := cc.cache.Get(jobcache.JobKeyByName(newPG.Namespace, newPG.Name))
        if err != nil &amp;&amp; newPG.Annotations != nil </span><span class="cov0" title="0">{
                klog.Warningf(
                        "Failed to find job in cache by PodGroup, this may not be a PodGroup for volcano job.")
        }</span>

        <span class="cov8" title="1">if newPG.Status.Phase != oldPG.Status.Phase </span><span class="cov8" title="1">{
                req := apis.Request{
                        Namespace: newPG.Namespace,
                        JobName:   newPG.Name,
                }
                switch newPG.Status.Phase </span>{
                case scheduling.PodGroupUnknown:<span class="cov0" title="0">
                        req.Event = bus.JobUnknownEvent</span>
                }
                <span class="cov8" title="1">key := jobhelpers.GetJobKeyByReq(&amp;req)
                queue := cc.getWorkerQueue(key)
                queue.Add(req)</span>
        }
}

// TODO(k82cn): add handler for PodGroup unschedulable event.
</pre>
		
		<pre class="file" id="file26" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "fmt"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/volcano/pkg/controllers/job/plugins"
        pluginsinterface "volcano.sh/volcano/pkg/controllers/job/plugins/interface"
)

func (cc *jobcontroller) pluginOnPodCreate(job *batch.Job, pod *v1.Pod) error <span class="cov8" title="1">{
        client := pluginsinterface.PluginClientset{KubeClients: cc.kubeClient}
        for name, args := range job.Spec.Plugins </span><span class="cov8" title="1">{
                pb, found := plugins.GetPluginBuilder(name)
                if !found </span><span class="cov8" title="1">{
                        err := fmt.Errorf("failed to get plugin %s", name)
                        klog.Error(err)
                        return err
                }</span>
                <span class="cov8" title="1">klog.Infof("Starting to execute plugin at &lt;pluginOnPodCreate&gt;: %s on job: &lt;%s/%s&gt;", name, job.Namespace, job.Name)
                if err := pb(client, args).OnPodCreate(pod, job); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to process on pod create plugin %s, err %v.", name, err)
                        return err
                }</span>
        }
        <span class="cov8" title="1">return nil</span>
}

func (cc *jobcontroller) pluginOnJobAdd(job *batch.Job) error <span class="cov8" title="1">{
        client := pluginsinterface.PluginClientset{KubeClients: cc.kubeClient}
        if job.Status.ControlledResources == nil </span><span class="cov8" title="1">{
                job.Status.ControlledResources = make(map[string]string)
        }</span>
        <span class="cov8" title="1">for name, args := range job.Spec.Plugins </span><span class="cov8" title="1">{
                pb, found := plugins.GetPluginBuilder(name)
                if !found </span><span class="cov8" title="1">{
                        err := fmt.Errorf("failed to get plugin %s", name)
                        klog.Error(err)
                        return err
                }</span>
                <span class="cov8" title="1">klog.Infof("Starting to execute plugin at &lt;pluginOnJobAdd&gt;: %s on job: &lt;%s/%s&gt;", name, job.Namespace, job.Name)
                if err := pb(client, args).OnJobAdd(job); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to process on job add plugin %s, err %v.", name, err)
                        return err
                }</span>
        }

        <span class="cov8" title="1">return nil</span>
}

func (cc *jobcontroller) pluginOnJobDelete(job *batch.Job) error <span class="cov8" title="1">{
        if job.Status.ControlledResources == nil </span><span class="cov8" title="1">{
                job.Status.ControlledResources = make(map[string]string)
        }</span>
        <span class="cov8" title="1">client := pluginsinterface.PluginClientset{KubeClients: cc.kubeClient}
        for name, args := range job.Spec.Plugins </span><span class="cov8" title="1">{
                pb, found := plugins.GetPluginBuilder(name)
                if !found </span><span class="cov8" title="1">{
                        err := fmt.Errorf("failed to get plugin %s", name)
                        klog.Error(err)
                        return err
                }</span>
                <span class="cov8" title="1">klog.Infof("Starting to execute plugin at &lt;pluginOnJobDelete&gt;: %s on job: &lt;%s/%s&gt;", name, job.Namespace, job.Name)
                if err := pb(client, args).OnJobDelete(job); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("failed to process on job delete plugin %s, err %v.", name, err)
                        return err
                }</span>
        }

        <span class="cov8" title="1">return nil</span>
}

func (cc *jobcontroller) pluginOnJobUpdate(job *batch.Job) error <span class="cov8" title="1">{
        client := pluginsinterface.PluginClientset{KubeClients: cc.kubeClient}
        if job.Status.ControlledResources == nil </span><span class="cov8" title="1">{
                job.Status.ControlledResources = make(map[string]string)
        }</span>
        <span class="cov8" title="1">for name, args := range job.Spec.Plugins </span><span class="cov0" title="0">{
                pb, found := plugins.GetPluginBuilder(name)
                if !found </span><span class="cov0" title="0">{
                        err := fmt.Errorf("failed to get plugin %s", name)
                        klog.Error(err)
                        return err
                }</span>
                <span class="cov0" title="0">klog.Infof("Starting to execute plugin at &lt;pluginOnJobUpdate&gt;: %s on job: &lt;%s/%s&gt;", name, job.Namespace, job.Name)
                if err := pb(client, args).OnJobUpdate(job); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to process on job update plugin %s, err %v.", name, err)
                        return err
                }</span>
        }

        <span class="cov8" title="1">return nil</span>
}
</pre>
		
		<pre class="file" id="file27" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"
        "time"

        "golang.org/x/time/rate"
        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"
)

func newRateLimitingQueue() workqueue.RateLimitingInterface <span class="cov8" title="1">{
        return workqueue.NewRateLimitingQueue(workqueue.NewMaxOfRateLimiter(
                workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 180*time.Second),
                // 10 qps, 100 bucket size.  This is only for retry speed and its only the overall factor (not per item)
                &amp;workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
        ))
}</span>

func (cc *jobcontroller) processResyncTask() <span class="cov0" title="0">{
        obj, shutdown := cc.errTasks.Get()
        if shutdown </span><span class="cov0" title="0">{
                return
        }</span>

        // one task only resync 10 times
        <span class="cov0" title="0">if cc.errTasks.NumRequeues(obj) &gt; 10 </span><span class="cov0" title="0">{
                cc.errTasks.Forget(obj)
                return
        }</span>

        <span class="cov0" title="0">defer cc.errTasks.Done(obj)

        task, ok := obj.(*v1.Pod)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("failed to convert %v to *v1.Pod", obj)
                return
        }</span>

        <span class="cov0" title="0">if err := cc.syncTask(task); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to sync pod &lt;%v/%v&gt;, retry it, err %v", task.Namespace, task.Name, err)
                cc.resyncTask(task)
        }</span>
}

func (cc *jobcontroller) syncTask(oldTask *v1.Pod) error <span class="cov0" title="0">{
        newPod, err := cc.kubeClient.CoreV1().Pods(oldTask.Namespace).Get(context.TODO(), oldTask.Name, metav1.GetOptions{})
        if err != nil </span><span class="cov0" title="0">{
                if errors.IsNotFound(err) </span><span class="cov0" title="0">{
                        if err := cc.cache.DeletePod(oldTask); err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("failed to delete cache pod &lt;%v/%v&gt;, err %v.", oldTask.Namespace, oldTask.Name, err)
                                return err
                        }</span>
                        <span class="cov0" title="0">klog.V(3).Infof("Pod &lt;%v/%v&gt; was deleted, removed from cache.", oldTask.Namespace, oldTask.Name)

                        return nil</span>
                }
                <span class="cov0" title="0">return fmt.Errorf("failed to get Pod &lt;%v/%v&gt;: err %v", oldTask.Namespace, oldTask.Name, err)</span>
        }

        <span class="cov0" title="0">return cc.cache.UpdatePod(newPod)</span>
}

func (cc *jobcontroller) resyncTask(task *v1.Pod) <span class="cov0" title="0">{
        cc.errTasks.AddRateLimited(task)
}</span>
</pre>
		
		<pre class="file" id="file28" style="display: none">/*
Copyright 2017 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "fmt"

        v1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/runtime/schema"
        "k8s.io/klog"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        schedulingv2 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/apis"
        jobhelpers "volcano.sh/volcano/pkg/controllers/job/helpers"
)

// MakePodName append podname,jobname,taskName and index and returns the string.
func MakePodName(jobName string, taskName string, index int) string <span class="cov8" title="1">{
        return fmt.Sprintf(jobhelpers.PodNameFmt, jobName, taskName, index)
}</span>

func createJobPod(job *batch.Job, template *v1.PodTemplateSpec, topologyPolicy batch.NumaPolicy, ix int, jobForwarding bool) *v1.Pod <span class="cov8" title="1">{
        templateCopy := template.DeepCopy()

        pod := &amp;v1.Pod{
                ObjectMeta: metav1.ObjectMeta{
                        Name:      jobhelpers.MakePodName(job.Name, template.Name, ix),
                        Namespace: job.Namespace,
                        OwnerReferences: []metav1.OwnerReference{
                                *metav1.NewControllerRef(job, helpers.JobKind),
                        },
                        Labels:      templateCopy.Labels,
                        Annotations: templateCopy.Annotations,
                },
                Spec: templateCopy.Spec,
        }

        // If no scheduler name in Pod, use scheduler name from Job.
        if len(pod.Spec.SchedulerName) == 0 </span><span class="cov8" title="1">{
                pod.Spec.SchedulerName = job.Spec.SchedulerName
        }</span>

        <span class="cov8" title="1">volumeMap := make(map[string]string)
        for _, volume := range job.Spec.Volumes </span><span class="cov8" title="1">{
                vcName := volume.VolumeClaimName
                name := fmt.Sprintf("%s-%s", job.Name, jobhelpers.GenRandomStr(12))
                if _, ok := volumeMap[vcName]; !ok </span><span class="cov8" title="1">{
                        volume := v1.Volume{
                                Name: name,
                                VolumeSource: v1.VolumeSource{
                                        PersistentVolumeClaim: &amp;v1.PersistentVolumeClaimVolumeSource{
                                                ClaimName: vcName,
                                        },
                                },
                        }
                        pod.Spec.Volumes = append(pod.Spec.Volumes, volume)
                        volumeMap[vcName] = name
                }</span> else<span class="cov0" title="0"> {
                        // duplicate volumes, should be prevented
                        continue</span>
                }

                <span class="cov8" title="1">for i, c := range pod.Spec.Containers </span><span class="cov8" title="1">{
                        vm := v1.VolumeMount{
                                MountPath: volume.MountPath,
                                Name:      name,
                        }
                        pod.Spec.Containers[i].VolumeMounts = append(c.VolumeMounts, vm)
                }</span>
        }

        <span class="cov8" title="1">tsKey := templateCopy.Name
        if len(tsKey) == 0 </span><span class="cov8" title="1">{
                tsKey = batch.DefaultTaskSpec
        }</span>

        <span class="cov8" title="1">if len(pod.Annotations) == 0 </span><span class="cov8" title="1">{
                pod.Annotations = make(map[string]string)
        }</span>

        <span class="cov8" title="1">pod.Annotations[batch.TaskSpecKey] = tsKey
        pod.Annotations[schedulingv2.KubeGroupNameAnnotationKey] = job.Name
        pod.Annotations[batch.JobNameKey] = job.Name
        pod.Annotations[batch.QueueNameKey] = job.Spec.Queue
        pod.Annotations[batch.JobVersion] = fmt.Sprintf("%d", job.Status.Version)
        pod.Annotations[batch.PodTemplateKey] = fmt.Sprintf("%s-%s", job.Name, template.Name)

        if topologyPolicy != "" </span><span class="cov0" title="0">{
                pod.Annotations[schedulingv2.NumaPolicyKey] = string(topologyPolicy)
        }</span>

        <span class="cov8" title="1">if len(job.Annotations) &gt; 0 </span><span class="cov0" title="0">{
                if value, found := job.Annotations[schedulingv2.PodPreemptable]; found </span><span class="cov0" title="0">{
                        pod.Annotations[schedulingv2.PodPreemptable] = value
                }</span>
                <span class="cov0" title="0">if value, found := job.Annotations[schedulingv2.RevocableZone]; found </span><span class="cov0" title="0">{
                        pod.Annotations[schedulingv2.RevocableZone] = value
                }</span>

                <span class="cov0" title="0">if value, found := job.Annotations[schedulingv2.JDBMinAvailable]; found </span><span class="cov0" title="0">{
                        pod.Annotations[schedulingv2.JDBMinAvailable] = value
                }</span> else<span class="cov0" title="0"> if value, found := job.Annotations[schedulingv2.JDBMaxUnavailable]; found </span><span class="cov0" title="0">{
                        pod.Annotations[schedulingv2.JDBMaxUnavailable] = value
                }</span>
        }

        <span class="cov8" title="1">if len(pod.Labels) == 0 </span><span class="cov8" title="1">{
                pod.Labels = make(map[string]string)
        }</span>

        // Set pod labels for Service.
        <span class="cov8" title="1">pod.Labels[batch.JobNameKey] = job.Name
        pod.Labels[batch.TaskSpecKey] = tsKey
        pod.Labels[batch.JobNamespaceKey] = job.Namespace
        pod.Labels[batch.QueueNameKey] = job.Spec.Queue
        if len(job.Labels) &gt; 0 </span><span class="cov0" title="0">{
                if value, found := job.Labels[schedulingv2.PodPreemptable]; found </span><span class="cov0" title="0">{
                        pod.Labels[schedulingv2.PodPreemptable] = value
                }</span>
        }

        <span class="cov8" title="1">if jobForwarding </span><span class="cov0" title="0">{
                pod.Annotations[batch.JobForwardingKey] = "true"
                pod.Labels[batch.JobForwardingKey] = "true"
        }</span>

        <span class="cov8" title="1">return pod</span>
}

func applyPolicies(job *batch.Job, req *apis.Request) v1alpha1.Action <span class="cov8" title="1">{
        if len(req.Action) != 0 </span><span class="cov8" title="1">{
                return req.Action
        }</span>

        <span class="cov8" title="1">if req.Event == v1alpha1.OutOfSyncEvent </span><span class="cov8" title="1">{
                return v1alpha1.SyncJobAction
        }</span>

        // For all the requests triggered from discarded job resources will perform sync action instead
        <span class="cov8" title="1">if req.JobVersion &lt; job.Status.Version </span><span class="cov0" title="0">{
                klog.Infof("Request %s is outdated, will perform sync instead.", req)
                return v1alpha1.SyncJobAction
        }</span>

        // Overwrite Job level policies
        <span class="cov8" title="1">if len(req.TaskName) != 0 </span><span class="cov8" title="1">{
                // Parse task level policies
                for _, task := range job.Spec.Tasks </span><span class="cov8" title="1">{
                        if task.Name == req.TaskName </span><span class="cov8" title="1">{
                                for _, policy := range task.Policies </span><span class="cov8" title="1">{
                                        policyEvents := getEventlist(policy)

                                        if len(policyEvents) &gt; 0 &amp;&amp; len(req.Event) &gt; 0 </span><span class="cov8" title="1">{
                                                if checkEventExist(policyEvents, req.Event) || checkEventExist(policyEvents, v1alpha1.AnyEvent) </span><span class="cov8" title="1">{
                                                        return policy.Action
                                                }</span>
                                        }

                                        // 0 is not an error code, is prevented in validation admission controller
                                        <span class="cov8" title="1">if policy.ExitCode != nil &amp;&amp; *policy.ExitCode == req.ExitCode </span><span class="cov8" title="1">{
                                                return policy.Action
                                        }</span>
                                }
                                <span class="cov8" title="1">break</span>
                        }
                }
        }

        // Parse Job level policies
        <span class="cov8" title="1">for _, policy := range job.Spec.Policies </span><span class="cov8" title="1">{
                policyEvents := getEventlist(policy)

                if len(policyEvents) &gt; 0 &amp;&amp; len(req.Event) &gt; 0 </span><span class="cov8" title="1">{
                        if checkEventExist(policyEvents, req.Event) || checkEventExist(policyEvents, v1alpha1.AnyEvent) </span><span class="cov8" title="1">{
                                return policy.Action
                        }</span>
                }

                // 0 is not an error code, is prevented in validation admission controller
                <span class="cov8" title="1">if policy.ExitCode != nil &amp;&amp; *policy.ExitCode == req.ExitCode </span><span class="cov8" title="1">{
                        return policy.Action
                }</span>
        }

        <span class="cov8" title="1">return v1alpha1.SyncJobAction</span>
}

func getEventlist(policy batch.LifecyclePolicy) []v1alpha1.Event <span class="cov8" title="1">{
        policyEventsList := policy.Events
        if len(policy.Event) &gt; 0 </span><span class="cov8" title="1">{
                policyEventsList = append(policyEventsList, policy.Event)
        }</span>
        <span class="cov8" title="1">return policyEventsList</span>
}

func checkEventExist(policyEvents []v1alpha1.Event, reqEvent v1alpha1.Event) bool <span class="cov8" title="1">{
        for _, event := range policyEvents </span><span class="cov8" title="1">{
                if event == reqEvent </span><span class="cov8" title="1">{
                        return true
                }</span>
        }
        <span class="cov0" title="0">return false</span>
}

func addResourceList(list, req, limit v1.ResourceList) <span class="cov8" title="1">{
        for name, quantity := range req </span><span class="cov8" title="1">{
                if value, ok := list[name]; !ok </span><span class="cov8" title="1">{
                        list[name] = quantity.DeepCopy()
                }</span> else<span class="cov8" title="1"> {
                        value.Add(quantity)
                        list[name] = value
                }</span>
        }

        <span class="cov8" title="1">if req != nil </span><span class="cov8" title="1">{
                return
        }</span>

        // If Requests is omitted for a container,
        // it defaults to Limits if that is explicitly specified.
        <span class="cov8" title="1">for name, quantity := range limit </span><span class="cov8" title="1">{
                if value, ok := list[name]; !ok </span><span class="cov8" title="1">{
                        list[name] = quantity.DeepCopy()
                }</span> else<span class="cov8" title="1"> {
                        value.Add(quantity)
                        list[name] = value
                }</span>
        }
}

// TaskPriority structure.
type TaskPriority struct {
        priority int32

        batch.TaskSpec
}

// TasksPriority is a slice of TaskPriority.
type TasksPriority []TaskPriority

func (p TasksPriority) Len() int <span class="cov8" title="1">{ return len(p) }</span>

func (p TasksPriority) Less(i, j int) bool <span class="cov8" title="1">{
        return p[i].priority &gt; p[j].priority
}</span>

func (p TasksPriority) Swap(i, j int) <span class="cov8" title="1">{ p[i], p[j] = p[j], p[i] }</span>

func isControlledBy(obj metav1.Object, gvk schema.GroupVersionKind) bool <span class="cov8" title="1">{
        controllerRef := metav1.GetControllerOf(obj)
        if controllerRef == nil </span><span class="cov0" title="0">{
                return false
        }</span>
        <span class="cov8" title="1">if controllerRef.APIVersion == gvk.GroupVersion().String() &amp;&amp; controllerRef.Kind == gvk.Kind </span><span class="cov8" title="1">{
                return true
        }</span>
        <span class="cov0" title="0">return false</span>
}
</pre>
		
		<pre class="file" id="file29" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ssh

import (
        "crypto/rand"
        "crypto/rsa"
        "crypto/x509"
        "encoding/pem"
        "flag"
        "fmt"

        "golang.org/x/crypto/ssh"
        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        jobhelpers "volcano.sh/volcano/pkg/controllers/job/helpers"
        pluginsinterface "volcano.sh/volcano/pkg/controllers/job/plugins/interface"
)

type sshPlugin struct {
        // Arguments given for the plugin
        pluginArguments []string

        client pluginsinterface.PluginClientset

        // flag parse args
        sshKeyFilePath string

        // private key string
        sshPrivateKey string

        // public key string
        sshPublicKey string
}

// New creates ssh plugin
func New(client pluginsinterface.PluginClientset, arguments []string) pluginsinterface.PluginInterface <span class="cov8" title="1">{
        p := sshPlugin{
                pluginArguments: arguments,
                client:          client,
                sshKeyFilePath:  SSHAbsolutePath,
        }

        p.addFlags()

        return &amp;p
}</span>

func (sp *sshPlugin) Name() string <span class="cov8" title="1">{
        return "ssh"
}</span>

func (sp *sshPlugin) OnPodCreate(pod *v1.Pod, job *batch.Job) error <span class="cov0" title="0">{
        sp.mountRsaKey(pod, job)

        return nil
}</span>

func (sp *sshPlugin) OnJobAdd(job *batch.Job) error <span class="cov0" title="0">{
        if job.Status.ControlledResources["plugin-"+sp.Name()] == sp.Name() </span><span class="cov0" title="0">{
                return nil
        }</span>

        <span class="cov0" title="0">var data map[string][]byte
        var err error
        if len(sp.sshPrivateKey) &gt; 0 </span><span class="cov0" title="0">{
                data, err = withUserProvidedRsaKey(job, sp.sshPrivateKey, sp.sshPublicKey)
        }</span> else<span class="cov0" title="0"> {
                data, err = generateRsaKey(job)
        }</span>
        <span class="cov0" title="0">if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">if err := helpers.CreateOrUpdateSecret(job, sp.client.KubeClients, data, sp.secretName(job)); err != nil </span><span class="cov0" title="0">{
                return fmt.Errorf("create secret for job &lt;%s/%s&gt; with ssh plugin failed for %v",
                        job.Namespace, job.Name, err)
        }</span>

        <span class="cov0" title="0">job.Status.ControlledResources["plugin-"+sp.Name()] = sp.Name()

        return nil</span>
}

func (sp *sshPlugin) OnJobDelete(job *batch.Job) error <span class="cov0" title="0">{
        if job.Status.ControlledResources["plugin-"+sp.Name()] != sp.Name() </span><span class="cov0" title="0">{
                return nil
        }</span>
        <span class="cov0" title="0">if err := helpers.DeleteSecret(job, sp.client.KubeClients, sp.secretName(job)); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>
        <span class="cov0" title="0">delete(job.Status.ControlledResources, "plugin-"+sp.Name())

        return nil</span>
}

// TODO: currently a container using a Secret as a subPath volume mount will not receive Secret updates.
// we may not update the job secret due to the above reason now.
// related issue: https://github.com/volcano-sh/volcano/issues/1420
func (sp *sshPlugin) OnJobUpdate(job *batch.Job) error <span class="cov0" title="0">{
        //data, err := generateRsaKey(job)
        //if err != nil {
        //        return err
        //}
        //
        //if err := helpers.CreateOrUpdateSecret(job, sp.client.KubeClients, data, sp.secretName(job)); err != nil {
        //        return fmt.Errorf("update secret for job &lt;%s/%s&gt; with ssh plugin failed for %v",
        //                job.Namespace, job.Name, err)
        //}

        return nil
}</span>

func (sp *sshPlugin) mountRsaKey(pod *v1.Pod, job *batch.Job) <span class="cov0" title="0">{
        secretName := sp.secretName(job)

        sshVolume := v1.Volume{
                Name: secretName,
        }

        var mode int32 = 0600
        sshVolume.Secret = &amp;v1.SecretVolumeSource{
                SecretName: secretName,
                Items: []v1.KeyToPath{
                        {
                                Key:  SSHPrivateKey,
                                Path: SSHRelativePath + "/" + SSHPrivateKey,
                        },
                        {
                                Key:  SSHPublicKey,
                                Path: SSHRelativePath + "/" + SSHPublicKey,
                        },
                        {
                                Key:  SSHAuthorizedKeys,
                                Path: SSHRelativePath + "/" + SSHAuthorizedKeys,
                        },
                        {
                                Key:  SSHConfig,
                                Path: SSHRelativePath + "/" + SSHConfig,
                        },
                },
                DefaultMode: &amp;mode,
        }

        if sp.sshKeyFilePath != SSHAbsolutePath </span><span class="cov0" title="0">{
                var noRootMode int32 = 0600
                sshVolume.Secret.DefaultMode = &amp;noRootMode
        }</span>

        <span class="cov0" title="0">pod.Spec.Volumes = append(pod.Spec.Volumes, sshVolume)

        for i, c := range pod.Spec.Containers </span><span class="cov0" title="0">{
                vm := v1.VolumeMount{
                        MountPath: sp.sshKeyFilePath,
                        SubPath:   SSHRelativePath,
                        Name:      secretName,
                }

                pod.Spec.Containers[i].VolumeMounts = append(c.VolumeMounts, vm)
        }</span>
        <span class="cov0" title="0">for i, c := range pod.Spec.InitContainers </span><span class="cov0" title="0">{
                vm := v1.VolumeMount{
                        MountPath: sp.sshKeyFilePath,
                        SubPath:   SSHRelativePath,
                        Name:      secretName,
                }

                pod.Spec.InitContainers[i].VolumeMounts = append(c.VolumeMounts, vm)
        }</span>
}

func generateRsaKey(job *batch.Job) (map[string][]byte, error) <span class="cov0" title="0">{
        bitSize := 1024

        privateKey, err := rsa.GenerateKey(rand.Reader, bitSize)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("rsa generateKey err: %v", err)
                return nil, err
        }</span>

        // id_rsa
        <span class="cov0" title="0">privBlock := pem.Block{
                Type:  "RSA PRIVATE KEY",
                Bytes: x509.MarshalPKCS1PrivateKey(privateKey),
        }
        privateKeyBytes := pem.EncodeToMemory(&amp;privBlock)

        // id_rsa.pub
        publicRsaKey, err := ssh.NewPublicKey(&amp;privateKey.PublicKey)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("ssh newPublicKey err: %v", err)
                return nil, err
        }</span>
        <span class="cov0" title="0">publicKeyBytes := ssh.MarshalAuthorizedKey(publicRsaKey)

        data := make(map[string][]byte)
        data[SSHPrivateKey] = privateKeyBytes
        data[SSHPublicKey] = publicKeyBytes
        data[SSHAuthorizedKeys] = publicKeyBytes
        data[SSHConfig] = []byte(generateSSHConfig(job))

        return data, nil</span>
}

func withUserProvidedRsaKey(job *batch.Job, sshPrivateKey string, sshPublicKey string) (map[string][]byte, error) <span class="cov0" title="0">{
        data := make(map[string][]byte)
        data[SSHPrivateKey] = []byte(sshPrivateKey)
        data[SSHPublicKey] = []byte(sshPublicKey)
        data[SSHAuthorizedKeys] = []byte(sshPublicKey)
        data[SSHConfig] = []byte(generateSSHConfig(job))

        return data, nil
}</span>

func (sp *sshPlugin) secretName(job *batch.Job) string <span class="cov0" title="0">{
        return fmt.Sprintf("%s-%s", job.Name, sp.Name())
}</span>

func (sp *sshPlugin) addFlags() <span class="cov8" title="1">{
        flagSet := flag.NewFlagSet(sp.Name(), flag.ContinueOnError)
        flagSet.StringVar(&amp;sp.sshKeyFilePath, "ssh-key-file-path", sp.sshKeyFilePath, "The path used to store "+
                "ssh private and public keys, it is `/root/.ssh` by default.")
        flagSet.StringVar(&amp;sp.sshPrivateKey, "ssh-private-key", sp.sshPrivateKey, "The input string of the private key")
        flagSet.StringVar(&amp;sp.sshPublicKey, "ssh-public-key", sp.sshPublicKey, "The input string of the public key")

        if err := flagSet.Parse(sp.pluginArguments); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("plugin %s flagset parse failed, err: %v", sp.Name(), err)
        }</span>
}

func generateSSHConfig(job *batch.Job) string <span class="cov0" title="0">{
        config := "StrictHostKeyChecking no\nUserKnownHostsFile /dev/null\n"

        for _, ts := range job.Spec.Tasks </span><span class="cov0" title="0">{
                for i := 0; i &lt; int(ts.Replicas); i++ </span><span class="cov0" title="0">{
                        hostName := ts.Template.Spec.Hostname
                        subdomain := ts.Template.Spec.Subdomain
                        if len(hostName) == 0 </span><span class="cov0" title="0">{
                                hostName = jobhelpers.MakePodName(job.Name, ts.Name, i)
                        }</span>
                        <span class="cov0" title="0">if len(subdomain) == 0 </span><span class="cov0" title="0">{
                                subdomain = job.Name
                        }</span>

                        <span class="cov0" title="0">config += "Host " + hostName + "\n"
                        config += "  HostName " + hostName + "." + subdomain + "\n"
                        if len(ts.Template.Spec.Hostname) != 0 </span><span class="cov0" title="0">{
                                break</span>
                        }
                }
        }

        <span class="cov0" title="0">return config</span>
}
</pre>
		
		<pre class="file" id="file30" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package podgroup

import (
        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/util/wait"
        coreinformers "k8s.io/client-go/informers/core/v1"
        "k8s.io/client-go/kubernetes"
        corelisters "k8s.io/client-go/listers/core/v1"
        "k8s.io/client-go/tools/cache"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"

        scheduling "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        vcclientset "volcano.sh/apis/pkg/client/clientset/versioned"
        informerfactory "volcano.sh/apis/pkg/client/informers/externalversions"
        schedulinginformer "volcano.sh/apis/pkg/client/informers/externalversions/scheduling/v1beta1"
        schedulinglister "volcano.sh/apis/pkg/client/listers/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/framework"
)

func init() <span class="cov8" title="1">{
        framework.RegisterController(&amp;pgcontroller{})
}</span>

// pgcontroller the Podgroup pgcontroller type.
type pgcontroller struct {
        kubeClient kubernetes.Interface
        vcClient   vcclientset.Interface

        podInformer coreinformers.PodInformer
        pgInformer  schedulinginformer.PodGroupInformer

        // A store of pods
        podLister corelisters.PodLister
        podSynced func() bool

        // A store of podgroups
        pgLister schedulinglister.PodGroupLister
        pgSynced func() bool

        queue workqueue.RateLimitingInterface
}

func (pg *pgcontroller) Name() string <span class="cov8" title="1">{
        return "pg-controller"
}</span>

// Initialize create new Podgroup Controller.
func (pg *pgcontroller) Initialize(opt *framework.ControllerOption) error <span class="cov8" title="1">{
        pg.kubeClient = opt.KubeClient
        pg.vcClient = opt.VolcanoClient

        pg.queue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())

        pg.podInformer = opt.SharedInformerFactory.Core().V1().Pods()
        pg.podLister = pg.podInformer.Lister()
        pg.podSynced = pg.podInformer.Informer().HasSynced
        pg.podInformer.Informer().AddEventHandler(
                cache.FilteringResourceEventHandler{
                        FilterFunc: func(obj interface{}) bool </span><span class="cov0" title="0">{
                                switch v := obj.(type) </span>{
                                case *v1.Pod:<span class="cov0" title="0">
                                        if v.Spec.SchedulerName == opt.SchedulerName &amp;&amp;
                                                (v.Annotations == nil || v.Annotations[scheduling.KubeGroupNameAnnotationKey] == "") </span><span class="cov0" title="0">{
                                                return true
                                        }</span>
                                        <span class="cov0" title="0">return false</span>
                                default:<span class="cov0" title="0">
                                        return false</span>
                                }
                        },
                        Handler: cache.ResourceEventHandlerFuncs{
                                AddFunc: pg.addPod,
                        },
                })

        <span class="cov8" title="1">pg.pgInformer = informerfactory.NewSharedInformerFactory(pg.vcClient, 0).Scheduling().V1beta1().PodGroups()
        pg.pgLister = pg.pgInformer.Lister()
        pg.pgSynced = pg.pgInformer.Informer().HasSynced

        return nil</span>
}

// Run start NewPodgroupController.
func (pg *pgcontroller) Run(stopCh &lt;-chan struct{}) <span class="cov0" title="0">{
        go pg.podInformer.Informer().Run(stopCh)
        go pg.pgInformer.Informer().Run(stopCh)

        cache.WaitForCacheSync(stopCh, pg.podSynced, pg.pgSynced)

        go wait.Until(pg.worker, 0, stopCh)

        klog.Infof("PodgroupController is running ...... ")
}</span>

func (pg *pgcontroller) worker() <span class="cov0" title="0">{
        for pg.processNextReq() </span>{<span class="cov0" title="0">
        }</span>
}

func (pg *pgcontroller) processNextReq() bool <span class="cov0" title="0">{
        obj, shutdown := pg.queue.Get()
        if shutdown </span><span class="cov0" title="0">{
                klog.Errorf("Fail to pop item from queue")
                return false
        }</span>

        <span class="cov0" title="0">req := obj.(podRequest)
        defer pg.queue.Done(req)

        pod, err := pg.podLister.Pods(req.podNamespace).Get(req.podName)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to get pod by &lt;%v&gt; from cache: %v", req, err)
                return true
        }</span>

        // normal pod use volcano
        <span class="cov0" title="0">if err := pg.createNormalPodPGIfNotExist(pod); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to handle Pod &lt;%s/%s&gt;: %v", pod.Namespace, pod.Name, err)
                pg.queue.AddRateLimited(req)
                return true
        }</span>

        // If no error, forget it.
        <span class="cov0" title="0">pg.queue.Forget(req)

        return true</span>
}
</pre>
		
		<pre class="file" id="file31" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package podgroup

import (
        "context"

        v1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/runtime/schema"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/helpers"
        scheduling "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)

type podRequest struct {
        podName      string
        podNamespace string
}

func (pg *pgcontroller) addPod(obj interface{}) <span class="cov8" title="1">{
        pod, ok := obj.(*v1.Pod)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Failed to convert %v to v1.Pod", obj)
                return
        }</span>

        <span class="cov8" title="1">req := podRequest{
                podName:      pod.Name,
                podNamespace: pod.Namespace,
        }

        pg.queue.Add(req)</span>
}

func (pg *pgcontroller) updatePodAnnotations(pod *v1.Pod, pgName string) error <span class="cov8" title="1">{
        if pod.Annotations == nil </span><span class="cov8" title="1">{
                pod.Annotations = make(map[string]string)
        }</span>
        <span class="cov8" title="1">if pod.Annotations[scheduling.KubeGroupNameAnnotationKey] == "" </span><span class="cov8" title="1">{
                pod.Annotations[scheduling.KubeGroupNameAnnotationKey] = pgName
        }</span> else<span class="cov0" title="0"> {
                if pod.Annotations[scheduling.KubeGroupNameAnnotationKey] != pgName </span><span class="cov0" title="0">{
                        klog.Errorf("normal pod %s/%s annotations %s value is not %s, but %s", pod.Namespace, pod.Name,
                                scheduling.KubeGroupNameAnnotationKey, pgName, pod.Annotations[scheduling.KubeGroupNameAnnotationKey])
                }</span>
                <span class="cov0" title="0">return nil</span>
        }

        <span class="cov8" title="1">if _, err := pg.kubeClient.CoreV1().Pods(pod.Namespace).Update(context.TODO(), pod, metav1.UpdateOptions{}); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to update pod &lt;%s/%s&gt;: %v", pod.Namespace, pod.Name, err)
                return err
        }</span>

        <span class="cov8" title="1">return nil</span>
}

func (pg *pgcontroller) createNormalPodPGIfNotExist(pod *v1.Pod) error <span class="cov8" title="1">{
        pgName := helpers.GeneratePodgroupName(pod)

        if _, err := pg.pgLister.PodGroups(pod.Namespace).Get(pgName); err != nil </span><span class="cov8" title="1">{
                if !apierrors.IsNotFound(err) </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to get normal PodGroup for Pod &lt;%s/%s&gt;: %v",
                                pod.Namespace, pod.Name, err)
                        return err
                }</span>

                <span class="cov8" title="1">obj := &amp;scheduling.PodGroup{
                        ObjectMeta: metav1.ObjectMeta{
                                Namespace:       pod.Namespace,
                                Name:            pgName,
                                OwnerReferences: newPGOwnerReferences(pod),
                                Annotations:     map[string]string{},
                                Labels:          map[string]string{},
                        },
                        Spec: scheduling.PodGroupSpec{
                                MinMember:         1,
                                PriorityClassName: pod.Spec.PriorityClassName,
                                MinResources:      calcPGMinResources(pod),
                        },
                }
                if queueName, ok := pod.Annotations[scheduling.QueueNameAnnotationKey]; ok </span><span class="cov0" title="0">{
                        obj.Spec.Queue = queueName
                }</span>

                <span class="cov8" title="1">if value, ok := pod.Annotations[scheduling.PodPreemptable]; ok </span><span class="cov0" title="0">{
                        obj.Annotations[scheduling.PodPreemptable] = value
                }</span>
                <span class="cov8" title="1">if value, ok := pod.Annotations[scheduling.RevocableZone]; ok </span><span class="cov0" title="0">{
                        obj.Annotations[scheduling.RevocableZone] = value
                }</span>
                <span class="cov8" title="1">if value, ok := pod.Labels[scheduling.PodPreemptable]; ok </span><span class="cov0" title="0">{
                        obj.Labels[scheduling.PodPreemptable] = value
                }</span>

                <span class="cov8" title="1">if value, found := pod.Annotations[scheduling.JDBMinAvailable]; found </span><span class="cov0" title="0">{
                        obj.Annotations[scheduling.JDBMinAvailable] = value
                }</span> else<span class="cov8" title="1"> if value, found := pod.Annotations[scheduling.JDBMaxUnavailable]; found </span><span class="cov0" title="0">{
                        obj.Annotations[scheduling.JDBMaxUnavailable] = value
                }</span>

                <span class="cov8" title="1">if _, err := pg.vcClient.SchedulingV1beta1().PodGroups(pod.Namespace).Create(context.TODO(), obj, metav1.CreateOptions{}); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to create normal PodGroup for Pod &lt;%s/%s&gt;: %v",
                                pod.Namespace, pod.Name, err)
                        return err
                }</span>
        }

        <span class="cov8" title="1">return pg.updatePodAnnotations(pod, pgName)</span>
}

func newPGOwnerReferences(pod *v1.Pod) []metav1.OwnerReference <span class="cov8" title="1">{
        if len(pod.OwnerReferences) != 0 </span><span class="cov8" title="1">{
                for _, ownerReference := range pod.OwnerReferences </span><span class="cov8" title="1">{
                        if ownerReference.Controller != nil &amp;&amp; *ownerReference.Controller </span><span class="cov8" title="1">{
                                return pod.OwnerReferences
                        }</span>
                }
        }

        <span class="cov8" title="1">gvk := schema.GroupVersionKind{
                Group:   v1.SchemeGroupVersion.Group,
                Version: v1.SchemeGroupVersion.Version,
                Kind:    "Pod",
        }
        ref := metav1.NewControllerRef(pod, gvk)
        return []metav1.OwnerReference{*ref}</span>
}

// addResourceList add list resource quantity
func addResourceList(list, req, limit v1.ResourceList) <span class="cov0" title="0">{
        for name, quantity := range req </span><span class="cov0" title="0">{
                if value, ok := list[name]; !ok </span><span class="cov0" title="0">{
                        list[name] = quantity.DeepCopy()
                }</span> else<span class="cov0" title="0"> {
                        value.Add(quantity)
                        list[name] = value
                }</span>
        }

        <span class="cov0" title="0">if req != nil </span><span class="cov0" title="0">{
                return
        }</span>

        // If Requests is omitted for a container,
        // it defaults to Limits if that is explicitly specified.
        <span class="cov0" title="0">for name, quantity := range limit </span><span class="cov0" title="0">{
                if value, ok := list[name]; !ok </span><span class="cov0" title="0">{
                        list[name] = quantity.DeepCopy()
                }</span> else<span class="cov0" title="0"> {
                        value.Add(quantity)
                        list[name] = value
                }</span>
        }
}

// calcPGMinResources calculate podgroup minimum resource
func calcPGMinResources(pod *v1.Pod) *v1.ResourceList <span class="cov8" title="1">{
        pgMinRes := v1.ResourceList{}

        for _, c := range pod.Spec.Containers </span><span class="cov0" title="0">{
                addResourceList(pgMinRes, c.Resources.Requests, c.Resources.Limits)
        }</span>

        <span class="cov8" title="1">return &amp;pgMinRes</span>
}
</pre>
		
		<pre class="file" id="file32" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"
        "sync"
        "time"

        v1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        utilruntime "k8s.io/apimachinery/pkg/util/runtime"
        "k8s.io/apimachinery/pkg/util/wait"
        "k8s.io/client-go/kubernetes"
        corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
        "k8s.io/client-go/tools/cache"
        "k8s.io/client-go/tools/record"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"

        busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        vcclientset "volcano.sh/apis/pkg/client/clientset/versioned"
        versionedscheme "volcano.sh/apis/pkg/client/clientset/versioned/scheme"
        informerfactory "volcano.sh/apis/pkg/client/informers/externalversions"
        busv1alpha1informer "volcano.sh/apis/pkg/client/informers/externalversions/bus/v1alpha1"
        schedulinginformer "volcano.sh/apis/pkg/client/informers/externalversions/scheduling/v1beta1"
        busv1alpha1lister "volcano.sh/apis/pkg/client/listers/bus/v1alpha1"
        schedulinglister "volcano.sh/apis/pkg/client/listers/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/apis"
        "volcano.sh/volcano/pkg/controllers/framework"
        queuestate "volcano.sh/volcano/pkg/controllers/queue/state"
)

func init() <span class="cov8" title="1">{
        framework.RegisterController(&amp;queuecontroller{})
}</span>

// queuecontroller manages queue status.
type queuecontroller struct {
        kubeClient kubernetes.Interface
        vcClient   vcclientset.Interface

        // informer
        queueInformer schedulinginformer.QueueInformer
        pgInformer    schedulinginformer.PodGroupInformer

        // queueLister
        queueLister schedulinglister.QueueLister
        queueSynced cache.InformerSynced

        // podGroup lister
        pgLister schedulinglister.PodGroupLister
        pgSynced cache.InformerSynced

        cmdInformer busv1alpha1informer.CommandInformer
        cmdLister   busv1alpha1lister.CommandLister
        cmdSynced   cache.InformerSynced

        // queues that need to be updated.
        queue        workqueue.RateLimitingInterface
        commandQueue workqueue.RateLimitingInterface

        pgMutex sync.RWMutex
        // queue name -&gt; podgroup namespace/name
        podGroups map[string]map[string]struct{}

        syncHandler        func(req *apis.Request) error
        syncCommandHandler func(cmd *busv1alpha1.Command) error

        enqueueQueue func(req *apis.Request)

        recorder      record.EventRecorder
        maxRequeueNum int
}

func (c *queuecontroller) Name() string <span class="cov8" title="1">{
        return "queue-controller"
}</span>

// NewQueueController creates a QueueController.
func (c *queuecontroller) Initialize(opt *framework.ControllerOption) error <span class="cov8" title="1">{
        c.vcClient = opt.VolcanoClient
        c.kubeClient = opt.KubeClient

        factory := informerfactory.NewSharedInformerFactory(c.vcClient, 0)
        queueInformer := factory.Scheduling().V1beta1().Queues()
        pgInformer := factory.Scheduling().V1beta1().PodGroups()

        eventBroadcaster := record.NewBroadcaster()
        eventBroadcaster.StartLogging(klog.Infof)
        eventBroadcaster.StartRecordingToSink(&amp;corev1.EventSinkImpl{Interface: c.kubeClient.CoreV1().Events("")})

        c.queueInformer = queueInformer
        c.pgInformer = pgInformer
        c.queueLister = queueInformer.Lister()
        c.queueSynced = queueInformer.Informer().HasSynced
        c.pgLister = pgInformer.Lister()
        c.pgSynced = pgInformer.Informer().HasSynced
        c.queue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
        c.commandQueue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
        c.podGroups = make(map[string]map[string]struct{})
        c.recorder = eventBroadcaster.NewRecorder(versionedscheme.Scheme, v1.EventSource{Component: "vc-controller-manager"})
        c.maxRequeueNum = opt.MaxRequeueNum
        if c.maxRequeueNum &lt; 0 </span><span class="cov0" title="0">{
                c.maxRequeueNum = -1
        }</span>

        <span class="cov8" title="1">queueInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    c.addQueue,
                UpdateFunc: c.updateQueue,
                DeleteFunc: c.deleteQueue,
        })

        pgInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    c.addPodGroup,
                UpdateFunc: c.updatePodGroup,
                DeleteFunc: c.deletePodGroup,
        })

        c.cmdInformer = informerfactory.NewSharedInformerFactory(c.vcClient, 0).Bus().V1alpha1().Commands()
        c.cmdInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
                FilterFunc: func(obj interface{}) bool </span><span class="cov0" title="0">{
                        switch v := obj.(type) </span>{
                        case *busv1alpha1.Command:<span class="cov0" title="0">
                                return IsQueueReference(v.TargetObject)</span>
                        default:<span class="cov0" title="0">
                                return false</span>
                        }
                },
                Handler: cache.ResourceEventHandlerFuncs{
                        AddFunc: c.addCommand,
                },
        })
        <span class="cov8" title="1">c.cmdLister = c.cmdInformer.Lister()
        c.cmdSynced = c.cmdInformer.Informer().HasSynced

        queuestate.SyncQueue = c.syncQueue
        queuestate.OpenQueue = c.openQueue
        queuestate.CloseQueue = c.closeQueue

        c.syncHandler = c.handleQueue
        c.syncCommandHandler = c.handleCommand

        c.enqueueQueue = c.enqueue

        return nil</span>
}

// Run starts QueueController.
func (c *queuecontroller) Run(stopCh &lt;-chan struct{}) <span class="cov0" title="0">{
        defer utilruntime.HandleCrash()
        defer c.queue.ShutDown()
        defer c.commandQueue.ShutDown()

        klog.Infof("Starting queue controller.")
        defer klog.Infof("Shutting down queue controller.")

        go c.queueInformer.Informer().Run(stopCh)
        go c.pgInformer.Informer().Run(stopCh)
        go c.cmdInformer.Informer().Run(stopCh)

        if !cache.WaitForCacheSync(stopCh, c.queueSynced, c.pgSynced, c.cmdSynced) </span><span class="cov0" title="0">{
                klog.Errorf("unable to sync caches for queue controller.")
                return
        }</span>

        <span class="cov0" title="0">go wait.Until(c.worker, 0, stopCh)
        go wait.Until(c.commandWorker, 0, stopCh)

        &lt;-stopCh</span>
}

// worker runs a worker thread that just dequeues items, processes them, and
// marks them done. You may run as many of these in parallel as you wish; the
// workqueue guarantees that they will not end up processing the same `queue`
// at the same time.
func (c *queuecontroller) worker() <span class="cov0" title="0">{
        for c.processNextWorkItem() </span>{<span class="cov0" title="0">
        }</span>
}

func (c *queuecontroller) processNextWorkItem() bool <span class="cov8" title="1">{
        obj, shutdown := c.queue.Get()
        if shutdown </span><span class="cov0" title="0">{
                return false
        }</span>
        <span class="cov8" title="1">defer c.queue.Done(obj)

        req, ok := obj.(*apis.Request)
        if !ok </span><span class="cov8" title="1">{
                klog.Errorf("%v is not a valid queue request struct.", obj)
                return true
        }</span>

        <span class="cov0" title="0">err := c.syncHandler(req)
        c.handleQueueErr(err, obj)

        return true</span>
}

func (c *queuecontroller) handleQueue(req *apis.Request) error <span class="cov0" title="0">{
        startTime := time.Now()
        defer func() </span><span class="cov0" title="0">{
                klog.V(4).Infof("Finished syncing queue %s (%v).", req.QueueName, time.Since(startTime))
        }</span>()

        <span class="cov0" title="0">queue, err := c.queueLister.Get(req.QueueName)
        if err != nil </span><span class="cov0" title="0">{
                if apierrors.IsNotFound(err) </span><span class="cov0" title="0">{
                        klog.V(4).Infof("Queue %s has been deleted.", req.QueueName)
                        return nil
                }</span>

                <span class="cov0" title="0">return fmt.Errorf("get queue %s failed for %v", req.QueueName, err)</span>
        }

        <span class="cov0" title="0">queueState := queuestate.NewState(queue)
        if queueState == nil </span><span class="cov0" title="0">{
                return fmt.Errorf("queue %s state %s is invalid", queue.Name, queue.Status.State)
        }</span>

        <span class="cov0" title="0">klog.V(4).Infof("Begin execute %s action for queue %s, current status %s", req.Action, req.QueueName, queue.Status.State)
        if err := queueState.Execute(req.Action); err != nil </span><span class="cov0" title="0">{
                return fmt.Errorf("sync queue %s failed for %v, event is %v, action is %s",
                        req.QueueName, err, req.Event, req.Action)
        }</span>

        <span class="cov0" title="0">return nil</span>
}

func (c *queuecontroller) handleQueueErr(err error, obj interface{}) <span class="cov0" title="0">{
        if err == nil </span><span class="cov0" title="0">{
                c.queue.Forget(obj)
                return
        }</span>

        <span class="cov0" title="0">if c.maxRequeueNum == -1 || c.queue.NumRequeues(obj) &lt; c.maxRequeueNum </span><span class="cov0" title="0">{
                klog.V(4).Infof("Error syncing queue request %v for %v.", obj, err)
                c.queue.AddRateLimited(obj)
                return
        }</span>

        <span class="cov0" title="0">req, _ := obj.(*apis.Request)
        c.recordEventsForQueue(req.QueueName, v1.EventTypeWarning, string(req.Action),
                fmt.Sprintf("%v queue failed for %v", req.Action, err))
        klog.V(2).Infof("Dropping queue request %v out of the queue for %v.", obj, err)
        c.queue.Forget(obj)</span>
}

func (c *queuecontroller) commandWorker() <span class="cov0" title="0">{
        for c.processNextCommand() </span>{<span class="cov0" title="0">
        }</span>
}

func (c *queuecontroller) processNextCommand() bool <span class="cov0" title="0">{
        obj, shutdown := c.commandQueue.Get()
        if shutdown </span><span class="cov0" title="0">{
                return false
        }</span>
        <span class="cov0" title="0">defer c.commandQueue.Done(obj)

        cmd, ok := obj.(*busv1alpha1.Command)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("%v is not a valid Command struct.", obj)
                return true
        }</span>

        <span class="cov0" title="0">err := c.syncCommandHandler(cmd)
        c.handleCommandErr(err, obj)

        return true</span>
}

func (c *queuecontroller) handleCommand(cmd *busv1alpha1.Command) error <span class="cov0" title="0">{
        startTime := time.Now()
        defer func() </span><span class="cov0" title="0">{
                klog.V(4).Infof("Finished syncing command %s/%s (%v).", cmd.Namespace, cmd.Name, time.Since(startTime))
        }</span>()

        <span class="cov0" title="0">err := c.vcClient.BusV1alpha1().Commands(cmd.Namespace).Delete(context.TODO(), cmd.Name, metav1.DeleteOptions{})
        if err != nil </span><span class="cov0" title="0">{
                if apierrors.IsNotFound(err) </span><span class="cov0" title="0">{
                        return nil
                }</span>

                <span class="cov0" title="0">return fmt.Errorf("failed to delete command &lt;%s/%s&gt; for %v", cmd.Namespace, cmd.Name, err)</span>
        }

        <span class="cov0" title="0">req := &amp;apis.Request{
                QueueName: cmd.TargetObject.Name,
                Event:     busv1alpha1.CommandIssuedEvent,
                Action:    busv1alpha1.Action(cmd.Action),
        }

        c.enqueueQueue(req)

        return nil</span>
}

func (c *queuecontroller) handleCommandErr(err error, obj interface{}) <span class="cov0" title="0">{
        if err == nil </span><span class="cov0" title="0">{
                c.commandQueue.Forget(obj)
                return
        }</span>

        <span class="cov0" title="0">if c.maxRequeueNum == -1 || c.commandQueue.NumRequeues(obj) &lt; c.maxRequeueNum </span><span class="cov0" title="0">{
                klog.V(4).Infof("Error syncing command %v for %v.", obj, err)
                c.commandQueue.AddRateLimited(obj)
                return
        }</span>

        <span class="cov0" title="0">klog.V(2).Infof("Dropping command %v out of the queue for %v.", obj, err)
        c.commandQueue.Forget(obj)</span>
}
</pre>
		
		<pre class="file" id="file33" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"
        "reflect"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/queue/state"

        v1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/client-go/tools/cache"

        "k8s.io/klog"
)

func (c *queuecontroller) syncQueue(queue *schedulingv1beta1.Queue, updateStateFn state.UpdateQueueStatusFn) error <span class="cov8" title="1">{
        klog.V(4).Infof("Begin to sync queue %s.", queue.Name)
        defer klog.V(4).Infof("End sync queue %s.", queue.Name)

        podGroups := c.getPodGroups(queue.Name)
        queueStatus := schedulingv1beta1.QueueStatus{}

        for _, pgKey := range podGroups </span><span class="cov8" title="1">{
                // Ignore error here, tt can not occur.
                ns, name, _ := cache.SplitMetaNamespaceKey(pgKey)

                // TODO: check NotFound error and sync local cache.
                pg, err := c.pgLister.PodGroups(ns).Get(name)
                if err != nil </span><span class="cov0" title="0">{
                        return err
                }</span>

                <span class="cov8" title="1">switch pg.Status.Phase </span>{
                case schedulingv1beta1.PodGroupPending:<span class="cov8" title="1">
                        queueStatus.Pending++</span>
                case schedulingv1beta1.PodGroupRunning:<span class="cov0" title="0">
                        queueStatus.Running++</span>
                case schedulingv1beta1.PodGroupUnknown:<span class="cov0" title="0">
                        queueStatus.Unknown++</span>
                case schedulingv1beta1.PodGroupInqueue:<span class="cov0" title="0">
                        queueStatus.Inqueue++</span>
                }
        }

        <span class="cov8" title="1">if updateStateFn != nil </span><span class="cov0" title="0">{
                updateStateFn(&amp;queueStatus, podGroups)
        }</span> else<span class="cov8" title="1"> {
                queueStatus.State = queue.Status.State
        }</span>

        // ignore update when status does not change
        <span class="cov8" title="1">if reflect.DeepEqual(queueStatus, queue.Status) </span><span class="cov0" title="0">{
                return nil
        }</span>

        <span class="cov8" title="1">newQueue := queue.DeepCopy()
        newQueue.Status = queueStatus
        if _, err := c.vcClient.SchedulingV1beta1().Queues().UpdateStatus(context.TODO(), newQueue, metav1.UpdateOptions{}); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to update status of Queue %s: %v.", newQueue.Name, err)
                return err
        }</span>

        <span class="cov8" title="1">return nil</span>
}

func (c *queuecontroller) openQueue(queue *schedulingv1beta1.Queue, updateStateFn state.UpdateQueueStatusFn) error <span class="cov0" title="0">{
        klog.V(4).Infof("Begin to open queue %s.", queue.Name)

        newQueue := queue.DeepCopy()
        newQueue.Status.State = schedulingv1beta1.QueueStateOpen

        if queue.Status.State != newQueue.Status.State </span><span class="cov0" title="0">{
                if _, err := c.vcClient.SchedulingV1beta1().Queues().Update(context.TODO(), newQueue, metav1.UpdateOptions{}); err != nil </span><span class="cov0" title="0">{
                        c.recorder.Event(newQueue, v1.EventTypeWarning, string(v1alpha1.OpenQueueAction),
                                fmt.Sprintf("Open queue failed for %v", err))
                        return err
                }</span>

                <span class="cov0" title="0">c.recorder.Event(newQueue, v1.EventTypeNormal, string(v1alpha1.OpenQueueAction), "Open queue succeed")</span>
        } else<span class="cov0" title="0"> {
                return nil
        }</span>

        <span class="cov0" title="0">q, err := c.vcClient.SchedulingV1beta1().Queues().Get(context.TODO(), newQueue.Name, metav1.GetOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">newQueue = q.DeepCopy()
        if updateStateFn != nil </span><span class="cov0" title="0">{
                updateStateFn(&amp;newQueue.Status, nil)
        }</span> else<span class="cov0" title="0"> {
                return fmt.Errorf("internal error, update state function should be provided")
        }</span>

        <span class="cov0" title="0">if queue.Status.State != newQueue.Status.State </span><span class="cov0" title="0">{
                if _, err := c.vcClient.SchedulingV1beta1().Queues().UpdateStatus(context.TODO(), newQueue, metav1.UpdateOptions{}); err != nil </span><span class="cov0" title="0">{
                        c.recorder.Event(newQueue, v1.EventTypeWarning, string(v1alpha1.OpenQueueAction),
                                fmt.Sprintf("Update queue status from %s to %s failed for %v",
                                        queue.Status.State, newQueue.Status.State, err))
                        return err
                }</span>
        }

        <span class="cov0" title="0">return nil</span>
}

func (c *queuecontroller) closeQueue(queue *schedulingv1beta1.Queue, updateStateFn state.UpdateQueueStatusFn) error <span class="cov0" title="0">{
        klog.V(4).Infof("Begin to close queue %s.", queue.Name)

        newQueue := queue.DeepCopy()
        newQueue.Status.State = schedulingv1beta1.QueueStateClosed

        if queue.Status.State != newQueue.Status.State </span><span class="cov0" title="0">{
                if _, err := c.vcClient.SchedulingV1beta1().Queues().Update(context.TODO(), newQueue, metav1.UpdateOptions{}); err != nil </span><span class="cov0" title="0">{
                        c.recorder.Event(newQueue, v1.EventTypeWarning, string(v1alpha1.CloseQueueAction),
                                fmt.Sprintf("Close queue failed for %v", err))
                        return err
                }</span>

                <span class="cov0" title="0">c.recorder.Event(newQueue, v1.EventTypeNormal, string(v1alpha1.CloseQueueAction), "Close queue succeed")</span>
        } else<span class="cov0" title="0"> {
                return nil
        }</span>

        <span class="cov0" title="0">q, err := c.vcClient.SchedulingV1beta1().Queues().Get(context.TODO(), newQueue.Name, metav1.GetOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">newQueue = q.DeepCopy()
        podGroups := c.getPodGroups(newQueue.Name)
        if updateStateFn != nil </span><span class="cov0" title="0">{
                updateStateFn(&amp;newQueue.Status, podGroups)
        }</span> else<span class="cov0" title="0"> {
                return fmt.Errorf("internal error, update state function should be provided")
        }</span>

        <span class="cov0" title="0">if queue.Status.State != newQueue.Status.State </span><span class="cov0" title="0">{
                if _, err := c.vcClient.SchedulingV1beta1().Queues().UpdateStatus(context.TODO(), newQueue, metav1.UpdateOptions{}); err != nil </span><span class="cov0" title="0">{
                        c.recorder.Event(newQueue, v1.EventTypeWarning, string(v1alpha1.CloseQueueAction),
                                fmt.Sprintf("Update queue status from %s to %s failed for %v",
                                        queue.Status.State, newQueue.Status.State, err))
                        return err
                }</span>
        }

        <span class="cov0" title="0">return nil</span>
}
</pre>
		
		<pre class="file" id="file34" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "k8s.io/client-go/tools/cache"
        "k8s.io/klog"

        busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/apis"
)

func (c *queuecontroller) enqueue(req *apis.Request) <span class="cov8" title="1">{
        c.queue.Add(req)
}</span>

func (c *queuecontroller) addQueue(obj interface{}) <span class="cov8" title="1">{
        queue := obj.(*schedulingv1beta1.Queue)

        req := &amp;apis.Request{
                QueueName: queue.Name,

                Event:  busv1alpha1.OutOfSyncEvent,
                Action: busv1alpha1.SyncQueueAction,
        }

        c.enqueue(req)
}</span>

func (c *queuecontroller) deleteQueue(obj interface{}) <span class="cov8" title="1">{
        queue, ok := obj.(*schedulingv1beta1.Queue)
        if !ok </span><span class="cov0" title="0">{
                tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Couldn't get object from tombstone %#v.", obj)
                        return
                }</span>
                <span class="cov0" title="0">queue, ok = tombstone.Obj.(*schedulingv1beta1.Queue)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Tombstone contained object that is not a Queue: %#v.", obj)
                        return
                }</span>
        }

        <span class="cov8" title="1">c.pgMutex.Lock()
        defer c.pgMutex.Unlock()
        delete(c.podGroups, queue.Name)</span>
}

func (c *queuecontroller) updateQueue(_, _ interface{}) {<span class="cov0" title="0">
        // currently do not care about queue update
}</span>

func (c *queuecontroller) addPodGroup(obj interface{}) <span class="cov8" title="1">{
        pg := obj.(*schedulingv1beta1.PodGroup)
        key, _ := cache.MetaNamespaceKeyFunc(obj)

        c.pgMutex.Lock()
        defer c.pgMutex.Unlock()

        if c.podGroups[pg.Spec.Queue] == nil </span><span class="cov8" title="1">{
                c.podGroups[pg.Spec.Queue] = make(map[string]struct{})
        }</span>
        <span class="cov8" title="1">c.podGroups[pg.Spec.Queue][key] = struct{}{}

        req := &amp;apis.Request{
                QueueName: pg.Spec.Queue,

                Event:  busv1alpha1.OutOfSyncEvent,
                Action: busv1alpha1.SyncQueueAction,
        }

        c.enqueue(req)</span>
}

func (c *queuecontroller) updatePodGroup(old, new interface{}) <span class="cov8" title="1">{
        oldPG := old.(*schedulingv1beta1.PodGroup)
        newPG := new.(*schedulingv1beta1.PodGroup)

        // Note: we have no use case update PodGroup.Spec.Queue
        // So do not consider it here.
        if oldPG.Status.Phase != newPG.Status.Phase </span><span class="cov8" title="1">{
                c.addPodGroup(newPG)
        }</span>
}

func (c *queuecontroller) deletePodGroup(obj interface{}) <span class="cov8" title="1">{
        pg, ok := obj.(*schedulingv1beta1.PodGroup)
        if !ok </span><span class="cov0" title="0">{
                tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Couldn't get object from tombstone %#v.", obj)
                        return
                }</span>
                <span class="cov0" title="0">pg, ok = tombstone.Obj.(*schedulingv1beta1.PodGroup)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Tombstone contained object that is not a PodGroup: %#v.", obj)
                        return
                }</span>
        }

        <span class="cov8" title="1">key, _ := cache.MetaNamespaceKeyFunc(obj)

        c.pgMutex.Lock()
        defer c.pgMutex.Unlock()

        delete(c.podGroups[pg.Spec.Queue], key)

        req := &amp;apis.Request{
                QueueName: pg.Spec.Queue,

                Event:  busv1alpha1.OutOfSyncEvent,
                Action: busv1alpha1.SyncQueueAction,
        }

        c.enqueue(req)</span>
}

func (c *queuecontroller) addCommand(obj interface{}) <span class="cov0" title="0">{
        cmd, ok := obj.(*busv1alpha1.Command)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Obj %v is not command.", obj)
                return
        }</span>

        <span class="cov0" title="0">c.commandQueue.Add(cmd)</span>
}

func (c *queuecontroller) getPodGroups(key string) []string <span class="cov8" title="1">{
        c.pgMutex.RLock()
        defer c.pgMutex.RUnlock()

        if c.podGroups[key] == nil </span><span class="cov0" title="0">{
                return nil
        }</span>
        <span class="cov8" title="1">podGroups := make([]string, 0, len(c.podGroups[key]))
        for pgKey := range c.podGroups[key] </span><span class="cov8" title="1">{
                podGroups = append(podGroups, pgKey)
        }</span>

        <span class="cov8" title="1">return podGroups</span>
}

func (c *queuecontroller) recordEventsForQueue(name, eventType, reason, message string) <span class="cov0" title="0">{
        queue, err := c.queueLister.Get(name)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Get queue %s failed for %v.", name, err)
                return
        }</span>

        <span class="cov0" title="0">c.recorder.Event(queue, eventType, reason, message)</span>
}
</pre>
		
		<pre class="file" id="file35" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// IsQueueReference return if ownerReference is Queue Kind.
func IsQueueReference(ref *metav1.OwnerReference) bool <span class="cov0" title="0">{
        if ref == nil </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov0" title="0">if ref.APIVersion != schedulingv1beta1.SchemeGroupVersion.String() </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov0" title="0">if ref.Kind != "Queue" </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov0" title="0">return true</span>
}
</pre>
		
		<pre class="file" id="file36" style="display: none">/*
 Copyright 2021 The Volcano Authors.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package allocate

import (
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/metrics"
        "volcano.sh/volcano/pkg/scheduler/util"
)

var targetJob = util.Reservation.TargetJob

type Action struct{}

func New() *Action <span class="cov8" title="1">{
        return &amp;Action{}
}</span>

func (alloc *Action) Name() string <span class="cov0" title="0">{
        return "allocate"
}</span>

func (alloc *Action) Initialize() {<span class="cov0" title="0">}</span>

func (alloc *Action) Execute(ssn *framework.Session) <span class="cov8" title="1">{
        klog.V(3).Infof("Enter Allocate ...")
        defer klog.V(3).Infof("Leaving Allocate ...")

        // the allocation for pod may have many stages
        // 1. pick a namespace named N (using ssn.NamespaceOrderFn)
        // 2. pick a queue named Q from N (using ssn.QueueOrderFn)
        // 3. pick a job named J from Q (using ssn.JobOrderFn)
        // 4. pick a task T from J (using ssn.TaskOrderFn)
        // 5. use predicateFn to filter out node that T can not be allocated on.
        // 6. use ssn.NodeOrderFn to judge the best node and assign it to T

        namespaces := util.NewPriorityQueue(ssn.NamespaceOrderFn)

        // jobsMap is map[api.NamespaceName]map[api.QueueID]PriorityQueue(*api.JobInfo)
        // used to find job with highest priority in given queue and namespace
        jobsMap := map[api.NamespaceName]map[api.QueueID]*util.PriorityQueue{}

        for _, job := range ssn.Jobs </span><span class="cov8" title="1">{
                if job.IsPending() </span><span class="cov0" title="0">{
                        klog.V(4).Infof("Job &lt;%s/%s&gt; Queue &lt;%s&gt; skip allocate, reason: job status is pending.",
                                job.Namespace, job.Name, job.Queue)
                        continue</span>
                }
                <span class="cov8" title="1">if vr := ssn.JobValid(job); vr != nil &amp;&amp; !vr.Pass </span><span class="cov0" title="0">{
                        klog.V(4).Infof("Job &lt;%s/%s&gt; Queue &lt;%s&gt; skip allocate, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message)
                        continue</span>
                }

                <span class="cov8" title="1">if _, found := ssn.Queues[job.Queue]; !found </span><span class="cov0" title="0">{
                        klog.Warningf("Skip adding Job &lt;%s/%s&gt; because its queue %s is not found",
                                job.Namespace, job.Name, job.Queue)
                        continue</span>
                }

                <span class="cov8" title="1">namespace := api.NamespaceName(job.Namespace)
                queueMap, found := jobsMap[namespace]
                if !found </span><span class="cov8" title="1">{
                        namespaces.Push(namespace)

                        queueMap = make(map[api.QueueID]*util.PriorityQueue)
                        jobsMap[namespace] = queueMap
                }</span>

                <span class="cov8" title="1">jobs, found := queueMap[job.Queue]
                if !found </span><span class="cov8" title="1">{
                        jobs = util.NewPriorityQueue(ssn.JobOrderFn)
                        queueMap[job.Queue] = jobs
                }</span>

                <span class="cov8" title="1">klog.V(4).Infof("Added Job &lt;%s/%s&gt; into Queue &lt;%s&gt;", job.Namespace, job.Name, job.Queue)
                jobs.Push(job)</span>
        }

        <span class="cov8" title="1">klog.V(3).Infof("Try to allocate resource to %d Namespaces", len(jobsMap))

        pendingTasks := map[api.JobID]*util.PriorityQueue{}

        allNodes := ssn.NodeList
        unlockedNodes := allNodes
        if targetJob != nil &amp;&amp; len(util.Reservation.LockedNodes) != 0 </span><span class="cov0" title="0">{
                unlockedNodes = unlockedNodes[0:0]
                for _, node := range allNodes </span><span class="cov0" title="0">{
                        if _, exist := util.Reservation.LockedNodes[node.Name]; !exist </span><span class="cov0" title="0">{
                                unlockedNodes = append(unlockedNodes, node)
                        }</span>
                }
        }
        <span class="cov8" title="1">for _, unlockedNode := range unlockedNodes </span><span class="cov8" title="1">{
                klog.V(4).Infof("unlockedNode ID: %s, Name: %s", unlockedNode.Node.UID, unlockedNode.Node.Name)
        }</span>
        <span class="cov8" title="1">predicateFn := func(task *api.TaskInfo, node *api.NodeInfo) error </span><span class="cov8" title="1">{
                // Check for Resource Predicate
                if !task.InitResreq.LessEqual(node.FutureIdle(), api.Zero) </span><span class="cov8" title="1">{
                        return api.NewFitError(task, node, api.NodeResourceFitFailed)
                }</span>

                <span class="cov8" title="1">return ssn.PredicateFn(task, node)</span>
        }

        // To pick &lt;namespace, queue&gt; tuple for job, we choose to pick namespace firstly.
        // Because we believe that number of queues would less than namespaces in most case.
        // And, this action would make the resource usage among namespace balanced.
        <span class="cov8" title="1">for </span><span class="cov8" title="1">{
                if namespaces.Empty() </span><span class="cov8" title="1">{
                        break</span>
                }

                // pick namespace from namespaces PriorityQueue
                <span class="cov8" title="1">namespace := namespaces.Pop().(api.NamespaceName)

                queueInNamespace := jobsMap[namespace]

                // pick queue for given namespace
                //
                // This block use a algorithm with time complex O(n).
                // But at least PriorityQueue could not be used here,
                // because the allocation of job would change the priority of queue among all namespaces,
                // and the PriorityQueue have no ability to update priority for a special queue.
                var queue *api.QueueInfo
                for queueID := range queueInNamespace </span><span class="cov8" title="1">{
                        currentQueue := ssn.Queues[queueID]
                        if ssn.Overused(currentQueue) </span><span class="cov8" title="1">{
                                klog.V(3).Infof("Namespace &lt;%s&gt; Queue &lt;%s&gt; is overused, ignore it.", namespace, currentQueue.Name)
                                delete(queueInNamespace, queueID)
                                continue</span>
                        }
                        <span class="cov8" title="1">if jobs, found := queueInNamespace[currentQueue.UID]; found &amp;&amp; jobs.Empty() </span><span class="cov8" title="1">{
                                continue</span>
                        }

                        <span class="cov8" title="1">if queue == nil || ssn.QueueOrderFn(currentQueue, queue) </span><span class="cov8" title="1">{
                                queue = currentQueue
                        }</span>
                }

                <span class="cov8" title="1">if queue == nil </span><span class="cov8" title="1">{
                        klog.V(3).Infof("Namespace &lt;%s&gt; have no queue, skip it", namespace)
                        continue</span>
                }

                <span class="cov8" title="1">klog.V(3).Infof("Try to allocate resource to Jobs in Namespace &lt;%s&gt; Queue &lt;%v&gt;", namespace, queue.Name)

                jobs, found := queueInNamespace[queue.UID]
                if !found || jobs.Empty() </span><span class="cov0" title="0">{
                        delete(queueInNamespace, queue.UID)
                        namespaces.Push(namespace)
                        klog.V(4).Infof("Can not find jobs for queue %s.", queue.Name)
                        continue</span>
                }

                <span class="cov8" title="1">job := jobs.Pop().(*api.JobInfo)
                var nodes []*api.NodeInfo
                if targetJob != nil &amp;&amp; job.UID == targetJob.UID </span><span class="cov0" title="0">{
                        klog.V(4).Infof("Try to allocate resource to target job: %s", job.Name)
                        nodes = allNodes
                }</span> else<span class="cov8" title="1"> {
                        nodes = unlockedNodes
                }</span>
                <span class="cov8" title="1">if _, found = pendingTasks[job.UID]; !found </span><span class="cov8" title="1">{
                        tasks := util.NewPriorityQueue(ssn.TaskOrderFn)
                        for _, task := range job.TaskStatusIndex[api.Pending] </span><span class="cov8" title="1">{
                                // Skip BestEffort task in 'allocate' action.
                                if task.Resreq.IsEmpty() </span><span class="cov0" title="0">{
                                        klog.V(4).Infof("Task &lt;%v/%v&gt; is BestEffort task, skip it.",
                                                task.Namespace, task.Name)
                                        continue</span>
                                }

                                <span class="cov8" title="1">tasks.Push(task)</span>
                        }
                        <span class="cov8" title="1">pendingTasks[job.UID] = tasks</span>
                }
                <span class="cov8" title="1">tasks := pendingTasks[job.UID]

                klog.V(3).Infof("Try to allocate resource to %d tasks of Job &lt;%v/%v&gt;",
                        tasks.Len(), job.Namespace, job.Name)

                stmt := framework.NewStatement(ssn)

                for !tasks.Empty() </span><span class="cov8" title="1">{
                        task := tasks.Pop().(*api.TaskInfo)

                        // Check whether the queue is overused on dimension that the task requested
                        taskRequest := task.Resreq.ResourceNames()
                        if underusedResources := ssn.UnderusedResources(queue); underusedResources != nil &amp;&amp; !underusedResources.Contains(taskRequest) </span><span class="cov8" title="1">{
                                klog.V(3).Infof("Queue &lt;%s&gt; is overused when considering task &lt;%s&gt;, ignore it.", queue.Name, task.Name)
                                continue</span>
                        }

                        <span class="cov8" title="1">klog.V(3).Infof("There are &lt;%d&gt; nodes for Job &lt;%v/%v&gt;", len(nodes), job.Namespace, job.Name)

                        predicateNodes, fitErrors := util.PredicateNodes(task, nodes, predicateFn)
                        if len(predicateNodes) == 0 </span><span class="cov8" title="1">{
                                job.NodesFitErrors[task.UID] = fitErrors
                                break</span>
                        }

                        <span class="cov8" title="1">var candidateNodes []*api.NodeInfo
                        for _, n := range predicateNodes </span><span class="cov8" title="1">{
                                if task.InitResreq.LessEqual(n.Idle, api.Zero) || task.InitResreq.LessEqual(n.FutureIdle(), api.Zero) </span><span class="cov8" title="1">{
                                        candidateNodes = append(candidateNodes, n)
                                }</span>
                        }

                        // If not candidate nodes for this task, skip it.
                        <span class="cov8" title="1">if len(candidateNodes) == 0 </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov8" title="1">nodeScores := util.PrioritizeNodes(task, candidateNodes, ssn.BatchNodeOrderFn, ssn.NodeOrderMapFn, ssn.NodeOrderReduceFn)

                        node := ssn.BestNodeFn(task, nodeScores)
                        if node == nil </span><span class="cov8" title="1">{
                                node = util.SelectBestNode(nodeScores)
                        }</span>

                        // Allocate idle resource to the task.
                        <span class="cov8" title="1">if task.InitResreq.LessEqual(node.Idle, api.Zero) </span><span class="cov8" title="1">{
                                klog.V(3).Infof("Binding Task &lt;%v/%v&gt; to node &lt;%v&gt;",
                                        task.Namespace, task.Name, node.Name)
                                if err := stmt.Allocate(task, node); err != nil </span><span class="cov0" title="0">{
                                        klog.Errorf("Failed to bind Task %v on %v in Session %v, err: %v",
                                                task.UID, node.Name, ssn.UID, err)
                                }</span> else<span class="cov8" title="1"> {
                                        metrics.UpdateE2eSchedulingDurationByJob(job.Name, string(job.Queue), job.Namespace, metrics.Duration(job.CreationTimestamp.Time))
                                }</span>
                        } else<span class="cov0" title="0"> {
                                klog.V(3).Infof("Predicates failed for task &lt;%s/%s&gt; on node &lt;%s&gt; with limited resources",
                                        task.Namespace, task.Name, node.Name)

                                // Allocate releasing resource to the task if any.
                                if task.InitResreq.LessEqual(node.FutureIdle(), api.Zero) </span><span class="cov0" title="0">{
                                        klog.V(3).Infof("Pipelining Task &lt;%v/%v&gt; to node &lt;%v&gt; for &lt;%v&gt; on &lt;%v&gt;",
                                                task.Namespace, task.Name, node.Name, task.InitResreq, node.Releasing)
                                        if err := stmt.Pipeline(task, node.Name); err != nil </span><span class="cov0" title="0">{
                                                klog.Errorf("Failed to pipeline Task %v on %v in Session %v for %v.",
                                                        task.UID, node.Name, ssn.UID, err)
                                        }</span> else<span class="cov0" title="0"> {
                                                metrics.UpdateE2eSchedulingDurationByJob(job.Name, string(job.Queue), job.Namespace, metrics.Duration(job.CreationTimestamp.Time))
                                        }</span>
                                }
                        }

                        <span class="cov8" title="1">if ssn.JobReady(job) &amp;&amp; !tasks.Empty() </span><span class="cov8" title="1">{
                                jobs.Push(job)
                                break</span>
                        }
                }

                <span class="cov8" title="1">if ssn.JobReady(job) </span><span class="cov8" title="1">{
                        stmt.Commit()
                }</span> else<span class="cov0" title="0"> {
                        if !ssn.JobPipelined(job) </span><span class="cov0" title="0">{
                                stmt.Discard()
                        }</span>
                }

                // Added Namespace back until no job in Namespace.
                <span class="cov8" title="1">namespaces.Push(namespace)</span>
        }
}

func (alloc *Action) UnInitialize() {<span class="cov0" title="0">}</span>
</pre>
		
		<pre class="file" id="file37" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package preempt

import (
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/metrics"
        "volcano.sh/volcano/pkg/scheduler/util"
)

type Action struct{}

func New() *Action <span class="cov8" title="1">{
        return &amp;Action{}
}</span>

func (alloc *Action) Name() string <span class="cov0" title="0">{
        return "preempt"
}</span>

func (alloc *Action) Initialize() {<span class="cov0" title="0">}</span>

func (alloc *Action) Execute(ssn *framework.Session) <span class="cov8" title="1">{
        klog.V(3).Infof("Enter Preempt ...")
        defer klog.V(3).Infof("Leaving Preempt ...")

        preemptorsMap := map[api.QueueID]*util.PriorityQueue{}
        preemptorTasks := map[api.JobID]*util.PriorityQueue{}

        var underRequest []*api.JobInfo
        queues := map[api.QueueID]*api.QueueInfo{}

        for _, job := range ssn.Jobs </span><span class="cov8" title="1">{
                if job.IsPending() </span><span class="cov0" title="0">{
                        continue</span>
                }

                <span class="cov8" title="1">if vr := ssn.JobValid(job); vr != nil &amp;&amp; !vr.Pass </span><span class="cov0" title="0">{
                        klog.V(4).Infof("Job &lt;%s/%s&gt; Queue &lt;%s&gt; skip preemption, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message)
                        continue</span>
                }

                <span class="cov8" title="1">if queue, found := ssn.Queues[job.Queue]; !found </span><span class="cov0" title="0">{
                        continue</span>
                } else<span class="cov8" title="1"> if _, existed := queues[queue.UID]; !existed </span><span class="cov8" title="1">{
                        klog.V(3).Infof("Added Queue &lt;%s&gt; for Job &lt;%s/%s&gt;",
                                queue.Name, job.Namespace, job.Name)
                        queues[queue.UID] = queue
                }</span>

                // check job if starting for more resources.
                <span class="cov8" title="1">if ssn.JobStarving(job) </span><span class="cov8" title="1">{
                        if _, found := preemptorsMap[job.Queue]; !found </span><span class="cov8" title="1">{
                                preemptorsMap[job.Queue] = util.NewPriorityQueue(ssn.JobOrderFn)
                        }</span>
                        <span class="cov8" title="1">preemptorsMap[job.Queue].Push(job)
                        underRequest = append(underRequest, job)
                        preemptorTasks[job.UID] = util.NewPriorityQueue(ssn.TaskOrderFn)
                        for _, task := range job.TaskStatusIndex[api.Pending] </span><span class="cov8" title="1">{
                                preemptorTasks[job.UID].Push(task)
                        }</span>
                }
        }

        // Preemption between Jobs within Queue.
        <span class="cov8" title="1">for _, queue := range queues </span><span class="cov8" title="1">{
                for </span><span class="cov8" title="1">{
                        preemptors := preemptorsMap[queue.UID]

                        // If no preemptors, no preemption.
                        if preemptors == nil || preemptors.Empty() </span><span class="cov8" title="1">{
                                klog.V(4).Infof("No preemptors in Queue &lt;%s&gt;, break.", queue.Name)
                                break</span>
                        }

                        <span class="cov8" title="1">preemptorJob := preemptors.Pop().(*api.JobInfo)

                        stmt := framework.NewStatement(ssn)
                        assigned := false
                        for </span><span class="cov8" title="1">{
                                // If job is not request more resource, then stop preempting.
                                if !ssn.JobStarving(preemptorJob) </span><span class="cov8" title="1">{
                                        break</span>
                                }

                                // If not preemptor tasks, next job.
                                <span class="cov8" title="1">if preemptorTasks[preemptorJob.UID].Empty() </span><span class="cov8" title="1">{
                                        klog.V(3).Infof("No preemptor task in job &lt;%s/%s&gt;.",
                                                preemptorJob.Namespace, preemptorJob.Name)
                                        break</span>
                                }

                                <span class="cov8" title="1">preemptor := preemptorTasks[preemptorJob.UID].Pop().(*api.TaskInfo)

                                if preempted, _ := preempt(ssn, stmt, preemptor, func(task *api.TaskInfo) bool </span><span class="cov8" title="1">{
                                        // Ignore non running task.
                                        if task.Status != api.Running </span><span class="cov0" title="0">{
                                                return false
                                        }</span>
                                        // Ignore task with empty resource request.
                                        <span class="cov8" title="1">if task.Resreq.IsEmpty() </span><span class="cov0" title="0">{
                                                return false
                                        }</span>
                                        <span class="cov8" title="1">job, found := ssn.Jobs[task.Job]
                                        if !found </span><span class="cov0" title="0">{
                                                return false
                                        }</span>
                                        // Preempt other jobs within queue
                                        <span class="cov8" title="1">return job.Queue == preemptorJob.Queue &amp;&amp; preemptor.Job != task.Job</span>
                                }); preempted <span class="cov8" title="1">{
                                        assigned = true
                                }</span>
                        }

                        // Commit changes only if job is pipelined, otherwise try next job.
                        <span class="cov8" title="1">if ssn.JobPipelined(preemptorJob) </span><span class="cov8" title="1">{
                                stmt.Commit()
                        }</span> else<span class="cov8" title="1"> {
                                stmt.Discard()
                                continue</span>
                        }

                        <span class="cov8" title="1">if assigned </span><span class="cov8" title="1">{
                                preemptors.Push(preemptorJob)
                        }</span>
                }

                // Preemption between Task within Job.
                <span class="cov8" title="1">for _, job := range underRequest </span><span class="cov8" title="1">{
                        // Fix: preemptor numbers lose when in same job
                        preemptorTasks[job.UID] = util.NewPriorityQueue(ssn.TaskOrderFn)
                        for _, task := range job.TaskStatusIndex[api.Pending] </span><span class="cov8" title="1">{
                                preemptorTasks[job.UID].Push(task)
                        }</span>
                        <span class="cov8" title="1">for </span><span class="cov8" title="1">{
                                if _, found := preemptorTasks[job.UID]; !found </span><span class="cov0" title="0">{
                                        break</span>
                                }

                                <span class="cov8" title="1">if preemptorTasks[job.UID].Empty() </span><span class="cov8" title="1">{
                                        break</span>
                                }

                                <span class="cov8" title="1">preemptor := preemptorTasks[job.UID].Pop().(*api.TaskInfo)

                                stmt := framework.NewStatement(ssn)
                                assigned, _ := preempt(ssn, stmt, preemptor, func(task *api.TaskInfo) bool </span><span class="cov8" title="1">{
                                        // Ignore non running task.
                                        if task.Status != api.Running </span><span class="cov8" title="1">{
                                                return false
                                        }</span>
                                        // Ignore task with empty resource request.
                                        <span class="cov8" title="1">if task.Resreq.IsEmpty() </span><span class="cov0" title="0">{
                                                return false
                                        }</span>
                                        // Preempt tasks within job.
                                        <span class="cov8" title="1">return preemptor.Job == task.Job</span>
                                })
                                <span class="cov8" title="1">stmt.Commit()

                                // If no preemption, next job.
                                if !assigned </span><span class="cov8" title="1">{
                                        break</span>
                                }
                        }
                }
        }

        // call victimTasksFn to evict tasks
        <span class="cov8" title="1">victimTasks(ssn)</span>
}

func (alloc *Action) UnInitialize() {<span class="cov0" title="0">}</span>

func preempt(
        ssn *framework.Session,
        stmt *framework.Statement,
        preemptor *api.TaskInfo,
        filter func(*api.TaskInfo) bool,
) (bool, error) <span class="cov8" title="1">{
        assigned := false

        allNodes := ssn.NodeList

        predicateNodes, _ := util.PredicateNodes(preemptor, allNodes, ssn.PredicateFn)

        nodeScores := util.PrioritizeNodes(preemptor, predicateNodes, ssn.BatchNodeOrderFn, ssn.NodeOrderMapFn, ssn.NodeOrderReduceFn)

        selectedNodes := util.SortNodes(nodeScores)
        for _, node := range selectedNodes </span><span class="cov8" title="1">{
                klog.V(3).Infof("Considering Task &lt;%s/%s&gt; on Node &lt;%s&gt;.",
                        preemptor.Namespace, preemptor.Name, node.Name)

                var preemptees []*api.TaskInfo
                for _, task := range node.Tasks </span><span class="cov8" title="1">{
                        if filter == nil </span><span class="cov0" title="0">{
                                preemptees = append(preemptees, task.Clone())
                        }</span> else<span class="cov8" title="1"> if filter(task) </span><span class="cov8" title="1">{
                                preemptees = append(preemptees, task.Clone())
                        }</span>
                }
                <span class="cov8" title="1">victims := ssn.Preemptable(preemptor, preemptees)
                metrics.UpdatePreemptionVictimsCount(len(victims))

                if err := util.ValidateVictims(preemptor, node, victims); err != nil </span><span class="cov8" title="1">{
                        klog.V(3).Infof("No validated victims on Node &lt;%s&gt;: %v", node.Name, err)
                        continue</span>
                }

                <span class="cov8" title="1">victimsQueue := util.NewPriorityQueue(func(l, r interface{}) bool </span><span class="cov8" title="1">{
                        return !ssn.TaskOrderFn(l, r)
                }</span>)
                <span class="cov8" title="1">for _, victim := range victims </span><span class="cov8" title="1">{
                        victimsQueue.Push(victim)
                }</span>
                // Preempt victims for tasks, pick lowest priority task first.
                <span class="cov8" title="1">preempted := api.EmptyResource()

                for !victimsQueue.Empty() </span><span class="cov8" title="1">{
                        // If reclaimed enough resources, break loop to avoid Sub panic.
                        if preemptor.InitResreq.LessEqual(node.FutureIdle(), api.Zero) </span><span class="cov0" title="0">{
                                break</span>
                        }
                        <span class="cov8" title="1">preemptee := victimsQueue.Pop().(*api.TaskInfo)
                        klog.V(3).Infof("Try to preempt Task &lt;%s/%s&gt; for Tasks &lt;%s/%s&gt;",
                                preemptee.Namespace, preemptee.Name, preemptor.Namespace, preemptor.Name)
                        if err := stmt.Evict(preemptee, "preempt"); err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("Failed to preempt Task &lt;%s/%s&gt; for Tasks &lt;%s/%s&gt;: %v",
                                        preemptee.Namespace, preemptee.Name, preemptor.Namespace, preemptor.Name, err)
                                continue</span>
                        }
                        <span class="cov8" title="1">preempted.Add(preemptee.Resreq)</span>
                }

                <span class="cov8" title="1">metrics.RegisterPreemptionAttempts()
                klog.V(3).Infof("Preempted &lt;%v&gt; for Task &lt;%s/%s&gt; requested &lt;%v&gt;.",
                        preempted, preemptor.Namespace, preemptor.Name, preemptor.InitResreq)

                if preemptor.InitResreq.LessEqual(node.FutureIdle(), api.Zero) </span><span class="cov8" title="1">{
                        if err := stmt.Pipeline(preemptor, node.Name); err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("Failed to pipeline Task &lt;%s/%s&gt; on Node &lt;%s&gt;",
                                        preemptor.Namespace, preemptor.Name, node.Name)
                        }</span>

                        // Ignore pipeline error, will be corrected in next scheduling loop.
                        <span class="cov8" title="1">assigned = true

                        break</span>
                }
        }

        <span class="cov8" title="1">return assigned, nil</span>
}

func victimTasks(ssn *framework.Session) <span class="cov8" title="1">{
        stmt := framework.NewStatement(ssn)
        victimTasks := ssn.VictimTasks()
        for _, victim := range victimTasks </span><span class="cov0" title="0">{
                if err := stmt.Evict(victim.Clone(), "evict"); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to evict Task &lt;%s/%s&gt;: %v",
                                victim.Namespace, victim.Name, err)
                        continue</span>
                }
        }
        <span class="cov8" title="1">stmt.Commit()</span>
}
</pre>
		
		<pre class="file" id="file38" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package reclaim

import (
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/util"
)

type Action struct{}

func New() *Action <span class="cov8" title="1">{
        return &amp;Action{}
}</span>

func (ra *Action) Name() string <span class="cov0" title="0">{
        return "reclaim"
}</span>

func (ra *Action) Initialize() {<span class="cov0" title="0">}</span>

func (ra *Action) Execute(ssn *framework.Session) <span class="cov8" title="1">{
        klog.V(3).Infof("Enter Reclaim ...")
        defer klog.V(3).Infof("Leaving Reclaim ...")

        queues := util.NewPriorityQueue(ssn.QueueOrderFn)
        queueMap := map[api.QueueID]*api.QueueInfo{}

        preemptorsMap := map[api.QueueID]*util.PriorityQueue{}
        preemptorTasks := map[api.JobID]*util.PriorityQueue{}

        klog.V(3).Infof("There are &lt;%d&gt; Jobs and &lt;%d&gt; Queues in total for scheduling.",
                len(ssn.Jobs), len(ssn.Queues))

        for _, job := range ssn.Jobs </span><span class="cov8" title="1">{
                if job.IsPending() </span><span class="cov0" title="0">{
                        continue</span>
                }

                <span class="cov8" title="1">if vr := ssn.JobValid(job); vr != nil &amp;&amp; !vr.Pass </span><span class="cov0" title="0">{
                        klog.V(4).Infof("Job &lt;%s/%s&gt; Queue &lt;%s&gt; skip reclaim, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message)
                        continue</span>
                }

                <span class="cov8" title="1">if queue, found := ssn.Queues[job.Queue]; !found </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to find Queue &lt;%s&gt; for Job &lt;%s/%s&gt;",
                                job.Queue, job.Namespace, job.Name)
                        continue</span>
                } else<span class="cov8" title="1"> if _, existed := queueMap[queue.UID]; !existed </span><span class="cov8" title="1">{
                        klog.V(4).Infof("Added Queue &lt;%s&gt; for Job &lt;%s/%s&gt;", queue.Name, job.Namespace, job.Name)
                        queueMap[queue.UID] = queue
                        queues.Push(queue)
                }</span>

                <span class="cov8" title="1">if len(job.TaskStatusIndex[api.Pending]) != 0 </span><span class="cov8" title="1">{
                        if _, found := preemptorsMap[job.Queue]; !found </span><span class="cov8" title="1">{
                                preemptorsMap[job.Queue] = util.NewPriorityQueue(ssn.JobOrderFn)
                        }</span>
                        <span class="cov8" title="1">preemptorsMap[job.Queue].Push(job)
                        preemptorTasks[job.UID] = util.NewPriorityQueue(ssn.TaskOrderFn)
                        for _, task := range job.TaskStatusIndex[api.Pending] </span><span class="cov8" title="1">{
                                preemptorTasks[job.UID].Push(task)
                        }</span>
                }
        }

        <span class="cov8" title="1">for </span><span class="cov8" title="1">{
                // If no queues, break
                if queues.Empty() </span><span class="cov8" title="1">{
                        break</span>
                }

                <span class="cov8" title="1">var job *api.JobInfo
                var task *api.TaskInfo

                queue := queues.Pop().(*api.QueueInfo)
                if ssn.Overused(queue) </span><span class="cov8" title="1">{
                        klog.V(3).Infof("Queue &lt;%s&gt; is overused, ignore it.", queue.Name)
                        continue</span>
                }

                // Found "high" priority job
                <span class="cov8" title="1">jobs, found := preemptorsMap[queue.UID]
                if !found || jobs.Empty() </span><span class="cov0" title="0">{
                        continue</span>
                } else<span class="cov8" title="1"> {
                        job = jobs.Pop().(*api.JobInfo)
                }</span>

                // Found "high" priority task to reclaim others
                <span class="cov8" title="1">if tasks, found := preemptorTasks[job.UID]; !found || tasks.Empty() </span><span class="cov0" title="0">{
                        continue</span>
                } else<span class="cov8" title="1"> {
                        task = tasks.Pop().(*api.TaskInfo)
                }</span>

                // Check whether the queue is overused on dimension that the task requested
                <span class="cov8" title="1">taskRequest := task.Resreq.ResourceNames()
                if underusedResources := ssn.UnderusedResources(queue); underusedResources != nil &amp;&amp; !underusedResources.Contains(taskRequest) </span><span class="cov0" title="0">{
                        klog.V(3).Infof("Queue &lt;%s&gt; is overused when considering task &lt;%s&gt;, ignore it.", queue.Name, task.Name)
                        continue</span>
                }

                <span class="cov8" title="1">assigned := false
                for _, n := range ssn.Nodes </span><span class="cov8" title="1">{
                        // If predicates failed, next node.
                        if err := ssn.PredicateFn(task, n); err != nil </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov8" title="1">klog.V(3).Infof("Considering Task &lt;%s/%s&gt; on Node &lt;%s&gt;.",
                                task.Namespace, task.Name, n.Name)

                        var reclaimees []*api.TaskInfo
                        for _, task := range n.Tasks </span><span class="cov8" title="1">{
                                // Ignore non running task.
                                if task.Status != api.Running </span><span class="cov0" title="0">{
                                        continue</span>
                                }

                                <span class="cov8" title="1">if j, found := ssn.Jobs[task.Job]; !found </span><span class="cov0" title="0">{
                                        continue</span>
                                } else<span class="cov8" title="1"> if j.Queue != job.Queue </span><span class="cov8" title="1">{
                                        q := ssn.Queues[j.Queue]
                                        if !q.Reclaimable() </span><span class="cov0" title="0">{
                                                continue</span>
                                        }
                                        // Clone task to avoid modify Task's status on node.
                                        <span class="cov8" title="1">reclaimees = append(reclaimees, task.Clone())</span>
                                }
                        }
                        <span class="cov8" title="1">victims := ssn.Reclaimable(task, reclaimees)

                        if err := util.ValidateVictims(task, n, victims); err != nil </span><span class="cov0" title="0">{
                                klog.V(3).Infof("No validated victims on Node &lt;%s&gt;: %v", n.Name, err)
                                continue</span>
                        }

                        <span class="cov8" title="1">resreq := task.InitResreq.Clone()
                        reclaimed := api.EmptyResource()

                        // Reclaim victims for tasks.
                        for _, reclaimee := range victims </span><span class="cov8" title="1">{
                                klog.Errorf("Try to reclaim Task &lt;%s/%s&gt; for Tasks &lt;%s/%s&gt;",
                                        reclaimee.Namespace, reclaimee.Name, task.Namespace, task.Name)
                                if err := ssn.Evict(reclaimee, "reclaim"); err != nil </span><span class="cov0" title="0">{
                                        klog.Errorf("Failed to reclaim Task &lt;%s/%s&gt; for Tasks &lt;%s/%s&gt;: %v",
                                                reclaimee.Namespace, reclaimee.Name, task.Namespace, task.Name, err)
                                        continue</span>
                                }
                                <span class="cov8" title="1">reclaimed.Add(reclaimee.Resreq)
                                // If reclaimed enough resources, break loop to avoid Sub panic.
                                if resreq.LessEqual(reclaimed, api.Zero) </span><span class="cov8" title="1">{
                                        break</span>
                                }
                        }

                        <span class="cov8" title="1">klog.V(3).Infof("Reclaimed &lt;%v&gt; for task &lt;%s/%s&gt; requested &lt;%v&gt;.",
                                reclaimed, task.Namespace, task.Name, task.InitResreq)

                        if task.InitResreq.LessEqual(reclaimed, api.Zero) </span><span class="cov8" title="1">{
                                if err := ssn.Pipeline(task, n.Name); err != nil </span><span class="cov0" title="0">{
                                        klog.Errorf("Failed to pipeline Task &lt;%s/%s&gt; on Node &lt;%s&gt;",
                                                task.Namespace, task.Name, n.Name)
                                }</span>

                                // Ignore error of pipeline, will be corrected in next scheduling loop.
                                <span class="cov8" title="1">assigned = true

                                break</span>
                        }
                }

                <span class="cov8" title="1">if assigned </span><span class="cov8" title="1">{
                        jobs.Push(job)
                }</span>
                <span class="cov8" title="1">queues.Push(queue)</span>
        }
}

func (ra *Action) UnInitialize() {<span class="cov0" title="0">
}</span>
</pre>
		
		<pre class="file" id="file39" style="display: none">/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "fmt"
)

// ClusterInfo is a snapshot of cluster by cache.
type ClusterInfo struct {
        Jobs           map[JobID]*JobInfo
        Nodes          map[string]*NodeInfo
        Queues         map[QueueID]*QueueInfo
        NamespaceInfo  map[NamespaceName]*NamespaceInfo
        RevocableNodes map[string]*NodeInfo
        NodeList       []string
}

func (ci ClusterInfo) String() string <span class="cov0" title="0">{
        str := "Cache:\n"

        if len(ci.Nodes) != 0 </span><span class="cov0" title="0">{
                str += "Nodes:\n"
                for _, n := range ci.Nodes </span><span class="cov0" title="0">{
                        str += fmt.Sprintf("\t %s: idle(%v) used(%v) allocatable(%v) pods(%d)\n",
                                n.Name, n.Idle, n.Used, n.Allocatable, len(n.Tasks))

                        i := 0
                        for _, p := range n.Tasks </span><span class="cov0" title="0">{
                                str += fmt.Sprintf("\t\t %d: %v\n", i, p)
                                i++
                        }</span>
                }
        }

        <span class="cov0" title="0">if len(ci.Jobs) != 0 </span><span class="cov0" title="0">{
                str += "Jobs:\n"
                for _, job := range ci.Jobs </span><span class="cov0" title="0">{
                        str += fmt.Sprintf("\t Job(%s) name(%s) minAvailable(%v)\n",
                                job.UID, job.Name, job.MinAvailable)

                        i := 0
                        for _, task := range job.Tasks </span><span class="cov0" title="0">{
                                str += fmt.Sprintf("\t\t %d: %v\n", i, task)
                                i++
                        }</span>
                }
        }

        <span class="cov0" title="0">if len(ci.NamespaceInfo) != 0 </span><span class="cov0" title="0">{
                str += "Namespaces:\n"
                for _, ns := range ci.NamespaceInfo </span><span class="cov0" title="0">{
                        str += fmt.Sprintf("\t Namespace(%s) Weight(%v)\n",
                                ns.Name, ns.Weight)
                }</span>
        }

        <span class="cov0" title="0">if len(ci.NodeList) != 0 </span><span class="cov0" title="0">{
                str += fmt.Sprintf("NodeList: %v\n", ci.NodeList)
        }</span>

        <span class="cov0" title="0">return str</span>
}
</pre>
		
		<pre class="file" id="file40" style="display: none">/*
Copyright 2020 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        v1 "k8s.io/api/core/v1"
)

// GPUDevice include gpu id, memory and the pods that are sharing it.
type GPUDevice struct {
        // GPU ID
        ID int
        // The pods that are sharing this GPU
        PodMap map[string]*v1.Pod
        // memory per card
        Memory uint
}

// NewGPUDevice creates a device
func NewGPUDevice(id int, mem uint) *GPUDevice <span class="cov0" title="0">{
        return &amp;GPUDevice{
                ID:     id,
                Memory: mem,
                PodMap: map[string]*v1.Pod{},
        }
}</span>

// getUsedGPUMemory calculates the used memory of the device.
func (g *GPUDevice) getUsedGPUMemory() uint <span class="cov0" title="0">{
        res := uint(0)
        for _, pod := range g.PodMap </span><span class="cov0" title="0">{
                if pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed </span><span class="cov0" title="0">{
                        continue</span>
                } else<span class="cov0" title="0"> {
                        gpuRequest := GetGPUResourceOfPod(pod)
                        res += gpuRequest
                }</span>
        }
        <span class="cov0" title="0">return res</span>
}

// GetGPUResourceOfPod returns the GPU resource required by the pod.
func GetGPUResourceOfPod(pod *v1.Pod) uint <span class="cov8" title="1">{
        var mem uint
        for _, container := range pod.Spec.Containers </span><span class="cov8" title="1">{
                mem += getGPUResourceOfContainer(&amp;container)
        }</span>
        <span class="cov8" title="1">return mem</span>
}

// getGPUResourceOfPod returns the GPU resource required by the container.
func getGPUResourceOfContainer(container *v1.Container) uint <span class="cov8" title="1">{
        var mem uint
        if val, ok := container.Resources.Limits[VolcanoGPUResource]; ok </span><span class="cov0" title="0">{
                mem = uint(val.Value())
        }</span>
        <span class="cov8" title="1">return mem</span>
}
</pre>
		
		<pre class="file" id="file41" style="display: none">/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "fmt"

        v1 "k8s.io/api/core/v1"
        clientcache "k8s.io/client-go/tools/cache"
)

// PodKey returns the string key of a pod.
func PodKey(pod *v1.Pod) TaskID <span class="cov8" title="1">{
        key, err := clientcache.MetaNamespaceKeyFunc(pod)
        if err != nil </span><span class="cov0" title="0">{
                return TaskID(fmt.Sprintf("%v/%v", pod.Namespace, pod.Name))
        }</span>
        <span class="cov8" title="1">return TaskID(key)</span>
}

func getTaskStatus(pod *v1.Pod) TaskStatus <span class="cov8" title="1">{
        switch pod.Status.Phase </span>{
        case v1.PodRunning:<span class="cov8" title="1">
                if pod.DeletionTimestamp != nil </span><span class="cov0" title="0">{
                        return Releasing
                }</span>

                <span class="cov8" title="1">return Running</span>
        case v1.PodPending:<span class="cov8" title="1">
                if pod.DeletionTimestamp != nil </span><span class="cov0" title="0">{
                        return Releasing
                }</span>

                <span class="cov8" title="1">if len(pod.Spec.NodeName) == 0 </span><span class="cov8" title="1">{
                        return Pending
                }</span>
                <span class="cov8" title="1">return Bound</span>
        case v1.PodUnknown:<span class="cov8" title="1">
                return Unknown</span>
        case v1.PodSucceeded:<span class="cov0" title="0">
                return Succeeded</span>
        case v1.PodFailed:<span class="cov0" title="0">
                return Failed</span>
        }

        <span class="cov0" title="0">return Unknown</span>
}

// AllocatedStatus checks whether the tasks has AllocatedStatus
func AllocatedStatus(status TaskStatus) bool <span class="cov8" title="1">{
        switch status </span>{
        case Bound, Binding, Running, Allocated:<span class="cov8" title="1">
                return true</span>
        default:<span class="cov8" title="1">
                return false</span>
        }
}

// MergeErrors is used to merge multiple errors into single error
func MergeErrors(errs ...error) error <span class="cov0" title="0">{
        msg := "errors: "

        foundErr := false
        i := 1

        for _, e := range errs </span><span class="cov0" title="0">{
                if e != nil </span><span class="cov0" title="0">{
                        if foundErr </span><span class="cov0" title="0">{
                                msg = fmt.Sprintf("%s, %d: ", msg, i)
                        }</span> else<span class="cov0" title="0"> {
                                msg = fmt.Sprintf("%s %d: ", msg, i)
                        }</span>

                        <span class="cov0" title="0">msg = fmt.Sprintf("%s%v", msg, e)
                        foundErr = true
                        i++</span>
                }
        }

        <span class="cov0" title="0">if foundErr </span><span class="cov0" title="0">{
                return fmt.Errorf("%s", msg)
        }</span>

        <span class="cov0" title="0">return nil</span>
}

// JobTerminated checks whether job was terminated.
func JobTerminated(job *JobInfo) bool <span class="cov0" title="0">{
        return job.PodGroup == nil &amp;&amp; len(job.Tasks) == 0
}</span>
</pre>
		
		<pre class="file" id="file42" style="display: none">/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "errors"
        "fmt"
        "sort"
        "strconv"
        "strings"
        "time"

        v1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/klog"
        volumescheduling "k8s.io/kubernetes/pkg/controller/volume/scheduling"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/apis/scheduling"
        "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)

// DisruptionBudget define job min pod available and max pod unvailable value
type DisruptionBudget struct {
        MinAvailable  string
        MaxUnavilable string
}

// NewDisruptionBudget create disruption budget for job
func NewDisruptionBudget(minAvailable, maxUnavilable string) *DisruptionBudget <span class="cov8" title="1">{
        disruptionBudget := &amp;DisruptionBudget{
                MinAvailable:  minAvailable,
                MaxUnavilable: maxUnavilable,
        }
        return disruptionBudget
}</span>

// Clone return a clone of DisruptionBudget
func (db *DisruptionBudget) Clone() *DisruptionBudget <span class="cov0" title="0">{
        return &amp;DisruptionBudget{
                MinAvailable:  db.MinAvailable,
                MaxUnavilable: db.MaxUnavilable,
        }
}</span>

// JobWaitingTime is maximum waiting time that a job could stay Pending in service level agreement
// when job waits longer than waiting time, it should be inqueue at once, and cluster should reserve resources for it
const JobWaitingTime = "sla-waiting-time"

// TaskID is UID type for Task
type TaskID types.UID

// TransactionContext holds all the fields that needed by scheduling transaction
type TransactionContext struct {
        NodeName string
        Status   TaskStatus
}

// Clone return a clone of TransactionContext
func (ctx *TransactionContext) Clone() *TransactionContext <span class="cov8" title="1">{
        if ctx == nil </span><span class="cov8" title="1">{
                return nil
        }</span>
        <span class="cov0" title="0">clone := *ctx
        return &amp;clone</span>
}

// TaskInfo will have all infos about the task
type TaskInfo struct {
        UID TaskID
        Job JobID

        Name      string
        Namespace string

        // Resreq is the resource that used when task running.
        Resreq *Resource
        // InitResreq is the resource that used to launch a task.
        InitResreq *Resource

        TransactionContext
        // LastTransaction holds the context of last scheduling transaction
        LastTransaction *TransactionContext

        Priority    int32
        VolumeReady bool
        Preemptable bool
        BestEffort  bool

        // RevocableZone support set volcano.sh/revocable-zone annotaion or label for pod/podgroup
        // we only support empty value or * value for this version and we will support specify revocable zone name for futrue release
        // empty value means workload can not use revocable node
        // * value means workload can use all the revocable node for during node active revocable time.
        RevocableZone string

        TopologyPolicy string
        PodVolumes     *volumescheduling.PodVolumes
        Pod            *v1.Pod
}

func getJobID(pod *v1.Pod) JobID <span class="cov8" title="1">{
        if gn, found := pod.Annotations[v1beta1.KubeGroupNameAnnotationKey]; found &amp;&amp; len(gn) != 0 </span><span class="cov8" title="1">{
                // Make sure Pod and PodGroup belong to the same namespace.
                jobID := fmt.Sprintf("%s/%s", pod.Namespace, gn)
                return JobID(jobID)
        }</span>

        <span class="cov8" title="1">return ""</span>
}

func getTaskID(pod *v1.Pod) TaskID <span class="cov0" title="0">{
        if ts, found := pod.Annotations[batch.TaskSpecKey]; found &amp;&amp; len(ts) != 0 </span><span class="cov0" title="0">{
                return TaskID(ts)
        }</span>

        <span class="cov0" title="0">return ""</span>
}

// NewTaskInfo creates new taskInfo object for a Pod
func NewTaskInfo(pod *v1.Pod) *TaskInfo <span class="cov8" title="1">{
        initResReq := GetPodResourceRequest(pod)
        resReq := initResReq
        bestEffort := initResReq.IsEmpty()
        preemptable := GetPodPreemptable(pod)
        revocableZone := GetPodRevocableZone(pod)
        topologyPolicy := GetPodTopologyPolicy(pod)

        jobID := getJobID(pod)

        ti := &amp;TaskInfo{
                UID:            TaskID(pod.UID),
                Job:            jobID,
                Name:           pod.Name,
                Namespace:      pod.Namespace,
                Priority:       1,
                Pod:            pod,
                Resreq:         resReq,
                InitResreq:     initResReq,
                Preemptable:    preemptable,
                BestEffort:     bestEffort,
                RevocableZone:  revocableZone,
                TopologyPolicy: topologyPolicy,

                TransactionContext: TransactionContext{
                        NodeName: pod.Spec.NodeName,
                        Status:   getTaskStatus(pod),
                },
        }

        if pod.Spec.Priority != nil </span><span class="cov0" title="0">{
                ti.Priority = *pod.Spec.Priority
        }</span>

        <span class="cov8" title="1">return ti</span>
}

// GetTransactionContext get transaction context of a task
func (ti *TaskInfo) GetTransactionContext() TransactionContext <span class="cov8" title="1">{
        return ti.TransactionContext
}</span>

// GenerateLastTxContext generate and set context of last transaction for a task
func (ti *TaskInfo) GenerateLastTxContext() <span class="cov0" title="0">{
        ctx := ti.GetTransactionContext()
        ti.LastTransaction = &amp;ctx
}</span>

// ClearLastTxContext clear context of last transaction for a task
func (ti *TaskInfo) ClearLastTxContext() <span class="cov0" title="0">{
        ti.LastTransaction = nil
}</span>

// Clone is used for cloning a task
func (ti *TaskInfo) Clone() *TaskInfo <span class="cov8" title="1">{
        return &amp;TaskInfo{
                UID:            ti.UID,
                Job:            ti.Job,
                Name:           ti.Name,
                Namespace:      ti.Namespace,
                Priority:       ti.Priority,
                PodVolumes:     ti.PodVolumes,
                Pod:            ti.Pod,
                Resreq:         ti.Resreq.Clone(),
                InitResreq:     ti.InitResreq.Clone(),
                VolumeReady:    ti.VolumeReady,
                Preemptable:    ti.Preemptable,
                BestEffort:     ti.BestEffort,
                RevocableZone:  ti.RevocableZone,
                TopologyPolicy: ti.TopologyPolicy,

                TransactionContext: TransactionContext{
                        NodeName: ti.NodeName,
                        Status:   ti.Status,
                },
                LastTransaction: ti.LastTransaction.Clone(),
        }
}</span>

// String returns the taskInfo details in a string
func (ti TaskInfo) String() string <span class="cov0" title="0">{
        return fmt.Sprintf("Task (%v:%v/%v): job %v, status %v, pri %v"+
                "resreq %v, preemptable %v, revocableZone %v, TopologyPolicy %v",
                ti.UID, ti.Namespace, ti.Name, ti.Job, ti.Status, ti.Priority,
                ti.Resreq, ti.Preemptable, ti.RevocableZone, ti.TopologyPolicy)
}</span>

// JobID is the type of JobInfo's ID.
type JobID types.UID

type tasksMap map[TaskID]*TaskInfo

// NodeResourceMap stores resource in a node
type NodeResourceMap map[string]*Resource

// JobInfo will have all info of a Job
type JobInfo struct {
        UID JobID

        Name      string
        Namespace string

        Queue QueueID

        Priority int32

        MinAvailable int32

        WaitingTime *time.Duration

        JobFitErrors   string
        NodesFitErrors map[TaskID]*FitErrors

        // All tasks of the Job.
        TaskStatusIndex       map[TaskStatus]tasksMap
        Tasks                 tasksMap
        TaskMinAvailable      map[TaskID]int32
        TaskMinAvailableTotal int32

        Allocated    *Resource
        TotalRequest *Resource

        CreationTimestamp metav1.Time
        PodGroup          *PodGroup

        ScheduleStartTimestamp metav1.Time

        Preemptable bool

        // RevocableZone support set volcano.sh/revocable-zone annotaion or label for pod/podgroup
        // we only support empty value or * value for this version and we will support specify revocable zone name for futrue release
        // empty value means workload can not use revocable node
        // * value means workload can use all the revocable node for during node active revocable time.
        RevocableZone string
        Budget        *DisruptionBudget
}

// NewJobInfo creates a new jobInfo for set of tasks
func NewJobInfo(uid JobID, tasks ...*TaskInfo) *JobInfo <span class="cov8" title="1">{
        job := &amp;JobInfo{
                UID:              uid,
                MinAvailable:     0,
                NodesFitErrors:   make(map[TaskID]*FitErrors),
                Allocated:        EmptyResource(),
                TotalRequest:     EmptyResource(),
                TaskStatusIndex:  map[TaskStatus]tasksMap{},
                Tasks:            tasksMap{},
                TaskMinAvailable: map[TaskID]int32{},
        }

        for _, task := range tasks </span><span class="cov0" title="0">{
                job.AddTaskInfo(task)
        }</span>

        <span class="cov8" title="1">return job</span>
}

// UnsetPodGroup removes podGroup details from a job
func (ji *JobInfo) UnsetPodGroup() <span class="cov0" title="0">{
        ji.PodGroup = nil
}</span>

// SetPodGroup sets podGroup details to a job
func (ji *JobInfo) SetPodGroup(pg *PodGroup) <span class="cov8" title="1">{
        ji.Name = pg.Name
        ji.Namespace = pg.Namespace
        ji.MinAvailable = pg.Spec.MinMember
        ji.Queue = QueueID(pg.Spec.Queue)
        ji.CreationTimestamp = pg.GetCreationTimestamp()

        var err error
        ji.WaitingTime, err = ji.extractWaitingTime(pg)
        if err != nil </span><span class="cov0" title="0">{
                klog.Warningf("Error occurs in parsing waiting time for job &lt;%s/%s&gt;, err: %s.",
                        pg.Namespace, pg.Name, err.Error())
                ji.WaitingTime = nil
        }</span>

        <span class="cov8" title="1">ji.Preemptable = ji.extractPreemptable(pg)
        ji.RevocableZone = ji.extractRevocableZone(pg)
        ji.Budget = ji.extractBudget(pg)

        taskMinAvailableTotal := int32(0)
        for task, member := range pg.Spec.MinTaskMember </span><span class="cov0" title="0">{
                ji.TaskMinAvailable[TaskID(task)] = member
                taskMinAvailableTotal += member
        }</span>
        <span class="cov8" title="1">ji.TaskMinAvailableTotal = taskMinAvailableTotal

        ji.PodGroup = pg</span>
}

// extractWaitingTime reads sla waiting time for job from podgroup annotations
// TODO: should also read from given field in volcano job spec
func (ji *JobInfo) extractWaitingTime(pg *PodGroup) (*time.Duration, error) <span class="cov8" title="1">{
        if _, exist := pg.Annotations[JobWaitingTime]; !exist </span><span class="cov8" title="1">{
                return nil, nil
        }</span>

        <span class="cov0" title="0">jobWaitingTime, err := time.ParseDuration(pg.Annotations[JobWaitingTime])
        if err != nil </span><span class="cov0" title="0">{
                return nil, err
        }</span>

        <span class="cov0" title="0">if jobWaitingTime &lt;= 0 </span><span class="cov0" title="0">{
                return nil, errors.New("invalid sla waiting time")
        }</span>

        <span class="cov0" title="0">return &amp;jobWaitingTime, nil</span>
}

// extractPreemptable return volcano.sh/preemptable value for job
func (ji *JobInfo) extractPreemptable(pg *PodGroup) bool <span class="cov8" title="1">{
        // check annotaion first
        if len(pg.Annotations) &gt; 0 </span><span class="cov0" title="0">{
                if value, found := pg.Annotations[v1beta1.PodPreemptable]; found </span><span class="cov0" title="0">{
                        b, err := strconv.ParseBool(value)
                        if err != nil </span><span class="cov0" title="0">{
                                klog.Warningf("invalid %s=%s", v1beta1.PodPreemptable, value)
                                return false
                        }</span>
                        <span class="cov0" title="0">return b</span>
                }
        }

        // it annotation does not exit, check label
        <span class="cov8" title="1">if len(pg.Labels) &gt; 0 </span><span class="cov0" title="0">{
                if value, found := pg.Labels[v1beta1.PodPreemptable]; found </span><span class="cov0" title="0">{
                        b, err := strconv.ParseBool(value)
                        if err != nil </span><span class="cov0" title="0">{
                                klog.Warningf("invalid %s=%s", v1beta1.PodPreemptable, value)
                                return false
                        }</span>
                        <span class="cov0" title="0">return b</span>
                }
        }

        <span class="cov8" title="1">return false</span>
}

// extractRevocableZone return volcano.sh/revocable-zone value for pod/podgroup
func (ji *JobInfo) extractRevocableZone(pg *PodGroup) string <span class="cov8" title="1">{
        // check annotaion first
        if len(pg.Annotations) &gt; 0 </span><span class="cov0" title="0">{
                if value, found := pg.Annotations[v1beta1.RevocableZone]; found </span><span class="cov0" title="0">{
                        if value != "*" </span><span class="cov0" title="0">{
                                return ""
                        }</span>
                        <span class="cov0" title="0">return value</span>
                }

                <span class="cov0" title="0">if value, found := pg.Annotations[v1beta1.PodPreemptable]; found </span><span class="cov0" title="0">{
                        if b, err := strconv.ParseBool(value); err == nil &amp;&amp; b </span><span class="cov0" title="0">{
                                return "*"
                        }</span>
                }
        }

        <span class="cov8" title="1">return ""</span>
}

// extractBudget return budget value for job
func (ji *JobInfo) extractBudget(pg *PodGroup) *DisruptionBudget <span class="cov8" title="1">{
        if len(pg.Annotations) &gt; 0 </span><span class="cov0" title="0">{
                if value, found := pg.Annotations[v1beta1.JDBMinAvailable]; found </span><span class="cov0" title="0">{
                        return NewDisruptionBudget(value, "")
                }</span> else<span class="cov0" title="0"> if value, found := pg.Annotations[v1beta1.JDBMaxUnavailable]; found </span><span class="cov0" title="0">{
                        return NewDisruptionBudget("", value)
                }</span>
        }

        <span class="cov8" title="1">return NewDisruptionBudget("", "")</span>
}

// GetMinResources return the min resources of podgroup.
func (ji *JobInfo) GetMinResources() *Resource <span class="cov0" title="0">{
        if ji.PodGroup.Spec.MinResources == nil </span><span class="cov0" title="0">{
                return EmptyResource()
        }</span>

        <span class="cov0" title="0">return NewResource(*ji.PodGroup.Spec.MinResources)</span>
}

func (ji *JobInfo) addTaskIndex(ti *TaskInfo) <span class="cov8" title="1">{
        if _, found := ji.TaskStatusIndex[ti.Status]; !found </span><span class="cov8" title="1">{
                ji.TaskStatusIndex[ti.Status] = tasksMap{}
        }</span>
        <span class="cov8" title="1">ji.TaskStatusIndex[ti.Status][ti.UID] = ti</span>
}

// AddTaskInfo is used to add a task to a job
func (ji *JobInfo) AddTaskInfo(ti *TaskInfo) <span class="cov8" title="1">{
        ji.Tasks[ti.UID] = ti
        ji.addTaskIndex(ti)
        ji.TotalRequest.Add(ti.Resreq)
        if AllocatedStatus(ti.Status) </span><span class="cov8" title="1">{
                ji.Allocated.Add(ti.Resreq)
        }</span>
}

// UpdateTaskStatus is used to update task's status in a job.
// If error occurs both task and job are guaranteed to be in the original state.
func (ji *JobInfo) UpdateTaskStatus(task *TaskInfo, status TaskStatus) error <span class="cov0" title="0">{
        if err := validateStatusUpdate(task.Status, status); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        // First remove the task (if exist) from the task list.
        <span class="cov0" title="0">if _, found := ji.Tasks[task.UID]; found </span><span class="cov0" title="0">{
                if err := ji.DeleteTaskInfo(task); err != nil </span><span class="cov0" title="0">{
                        return err
                }</span>
        }

        // Update task's status to the target status once task addition is guaranteed to succeed.
        <span class="cov0" title="0">task.Status = status
        ji.AddTaskInfo(task)

        return nil</span>
}

func (ji *JobInfo) deleteTaskIndex(ti *TaskInfo) <span class="cov8" title="1">{
        if tasks, found := ji.TaskStatusIndex[ti.Status]; found </span><span class="cov8" title="1">{
                delete(tasks, ti.UID)

                if len(tasks) == 0 </span><span class="cov8" title="1">{
                        delete(ji.TaskStatusIndex, ti.Status)
                }</span>
        }
}

// DeleteTaskInfo is used to delete a task from a job
func (ji *JobInfo) DeleteTaskInfo(ti *TaskInfo) error <span class="cov8" title="1">{
        if task, found := ji.Tasks[ti.UID]; found </span><span class="cov8" title="1">{
                ji.TotalRequest.Sub(task.Resreq)
                if AllocatedStatus(task.Status) </span><span class="cov8" title="1">{
                        ji.Allocated.Sub(task.Resreq)
                }</span>
                <span class="cov8" title="1">delete(ji.Tasks, task.UID)
                ji.deleteTaskIndex(task)
                return nil</span>
        }

        <span class="cov0" title="0">return fmt.Errorf("failed to find task &lt;%v/%v&gt; in job &lt;%v/%v&gt;",
                ti.Namespace, ti.Name, ji.Namespace, ji.Name)</span>
}

// Clone is used to clone a jobInfo object
func (ji *JobInfo) Clone() *JobInfo <span class="cov0" title="0">{
        info := &amp;JobInfo{
                UID:       ji.UID,
                Name:      ji.Name,
                Namespace: ji.Namespace,
                Queue:     ji.Queue,
                Priority:  ji.Priority,

                MinAvailable:   ji.MinAvailable,
                WaitingTime:    ji.WaitingTime,
                JobFitErrors:   ji.JobFitErrors,
                NodesFitErrors: make(map[TaskID]*FitErrors),
                Allocated:      EmptyResource(),
                TotalRequest:   EmptyResource(),

                PodGroup: ji.PodGroup,

                TaskStatusIndex:       map[TaskStatus]tasksMap{},
                TaskMinAvailable:      ji.TaskMinAvailable,
                TaskMinAvailableTotal: ji.TaskMinAvailableTotal,
                Tasks:                 tasksMap{},
                Preemptable:           ji.Preemptable,
                RevocableZone:         ji.RevocableZone,
                Budget:                ji.Budget.Clone(),
        }

        ji.CreationTimestamp.DeepCopyInto(&amp;info.CreationTimestamp)

        for _, task := range ji.Tasks </span><span class="cov0" title="0">{
                info.AddTaskInfo(task.Clone())
        }</span>

        <span class="cov0" title="0">return info</span>
}

// String returns a jobInfo object in string format
func (ji JobInfo) String() string <span class="cov0" title="0">{
        res := ""

        i := 0
        for _, task := range ji.Tasks </span><span class="cov0" title="0">{
                res += fmt.Sprintf("\n\t %d: %v", i, task)
                i++
        }</span>

        <span class="cov0" title="0">return fmt.Sprintf("Job (%v): namespace %v (%v), name %v, minAvailable %d, podGroup %+v, preemptable %+v, revocableZone %+v, minAvailable %+v, maxAvailable %+v",
                ji.UID, ji.Namespace, ji.Queue, ji.Name, ji.MinAvailable, ji.PodGroup, ji.Preemptable, ji.RevocableZone, ji.Budget.MinAvailable, ji.Budget.MaxUnavilable) + res</span>
}

// FitError returns detailed information on why a job's task failed to fit on
// each available node
func (ji *JobInfo) FitError() string <span class="cov8" title="1">{
        sortReasonsHistogram := func(reasons map[string]int) []string </span><span class="cov8" title="1">{
                reasonStrings := []string{}
                for k, v := range reasons </span><span class="cov8" title="1">{
                        reasonStrings = append(reasonStrings, fmt.Sprintf("%v %v", v, k))
                }</span>
                <span class="cov8" title="1">sort.Strings(reasonStrings)
                return reasonStrings</span>
        }

        // Stat histogram for all tasks of the job
        <span class="cov8" title="1">reasons := make(map[string]int)
        for status, taskMap := range ji.TaskStatusIndex </span><span class="cov8" title="1">{
                reasons[status.String()] += len(taskMap)
        }</span>
        <span class="cov8" title="1">reasons["minAvailable"] = int(ji.MinAvailable)
        reasonMsg := fmt.Sprintf("%v, %v", scheduling.PodGroupNotReady, strings.Join(sortReasonsHistogram(reasons), ", "))

        // Stat histogram for pending tasks only
        reasons = make(map[string]int)
        for uid := range ji.TaskStatusIndex[Pending] </span><span class="cov8" title="1">{
                reason, _ := ji.TaskSchedulingReason(uid)
                reasons[reason]++
        }</span>
        <span class="cov8" title="1">if len(reasons) &gt; 0 </span><span class="cov8" title="1">{
                reasonMsg += "; " + fmt.Sprintf("%s: %s", Pending.String(), strings.Join(sortReasonsHistogram(reasons), ", "))
        }</span>
        <span class="cov8" title="1">return reasonMsg</span>
}

// TaskSchedulingReason get detailed reason and message of the given task
// It returns detailed reason and message for tasks based on last scheduling transaction.
func (ji *JobInfo) TaskSchedulingReason(tid TaskID) (reason string, msg string) <span class="cov8" title="1">{
        taskInfo, exists := ji.Tasks[tid]
        if !exists </span><span class="cov0" title="0">{
                return "", ""
        }</span>

        // Get detailed scheduling reason based on LastTransaction
        <span class="cov8" title="1">ctx := taskInfo.GetTransactionContext()
        if taskInfo.LastTransaction != nil </span><span class="cov8" title="1">{
                ctx = *taskInfo.LastTransaction
        }</span>

        <span class="cov8" title="1">msg = ji.JobFitErrors
        switch status := ctx.Status; status </span>{
        case Allocated, Pipelined:<span class="cov8" title="1">
                // Pod is schedulable
                msg = fmt.Sprintf("Pod %s/%s can possibly be assigned to %s", taskInfo.Namespace, taskInfo.Name, ctx.NodeName)
                if status == Pipelined </span><span class="cov0" title="0">{
                        msg += " once resource is released"
                }</span>
                <span class="cov8" title="1">return PodReasonSchedulable, msg</span>
        case Pending:<span class="cov8" title="1">
                if fe := ji.NodesFitErrors[tid]; fe != nil </span><span class="cov8" title="1">{
                        // Pod is not schedulable
                        return PodReasonUnschedulable, fe.Error()
                }</span>
                // Pod is not scheduled yet
                <span class="cov8" title="1">return PodReasonUndetermined, msg</span>
        default:<span class="cov0" title="0">
                return status.String(), msg</span>
        }
}

// ReadyTaskNum returns the number of tasks that are ready or that is best-effort.
func (ji *JobInfo) ReadyTaskNum() int32 <span class="cov0" title="0">{
        occupied := 0
        occupied += len(ji.TaskStatusIndex[Bound])
        occupied += len(ji.TaskStatusIndex[Binding])
        occupied += len(ji.TaskStatusIndex[Running])
        occupied += len(ji.TaskStatusIndex[Allocated])
        occupied += len(ji.TaskStatusIndex[Succeeded])

        if tasks, found := ji.TaskStatusIndex[Pending]; found </span><span class="cov0" title="0">{
                for _, task := range tasks </span><span class="cov0" title="0">{
                        if task.BestEffort </span><span class="cov0" title="0">{
                                occupied++
                        }</span>
                }
        }

        <span class="cov0" title="0">return int32(occupied)</span>
}

// WaitingTaskNum returns the number of tasks that are pipelined.
func (ji *JobInfo) WaitingTaskNum() int32 <span class="cov0" title="0">{
        return int32(len(ji.TaskStatusIndex[Pipelined]))
}</span>

// CheckTaskMinAvailable returns whether each task of job is valid.
func (ji *JobInfo) CheckTaskMinAvailable() bool <span class="cov0" title="0">{
        // if job minAvailable is less than sumof(task minAvailable), skip this check
        if ji.MinAvailable &lt; ji.TaskMinAvailableTotal </span><span class="cov0" title="0">{
                return true
        }</span>

        <span class="cov0" title="0">actual := map[TaskID]int32{}
        for status, tasks := range ji.TaskStatusIndex </span><span class="cov0" title="0">{
                if AllocatedStatus(status) ||
                        status == Succeeded ||
                        status == Pipelined ||
                        status == Pending </span><span class="cov0" title="0">{
                        for _, task := range tasks </span><span class="cov0" title="0">{
                                actual[getTaskID(task.Pod)]++
                        }</span>
                }
        }

        <span class="cov0" title="0">klog.V(4).Infof("job %s/%s actual: %+v, ji.TaskMinAvailable: %+v", ji.Name, ji.Namespace, actual, ji.TaskMinAvailable)
        for task, minAvailable := range ji.TaskMinAvailable </span><span class="cov0" title="0">{
                if act, ok := actual[task]; !ok || act &lt; minAvailable </span><span class="cov0" title="0">{
                        return false
                }</span>
        }

        <span class="cov0" title="0">return true</span>
}

// ValidTaskNum returns the number of tasks that are valid.
func (ji *JobInfo) ValidTaskNum() int32 <span class="cov0" title="0">{
        occupied := 0
        for status, tasks := range ji.TaskStatusIndex </span><span class="cov0" title="0">{
                if AllocatedStatus(status) ||
                        status == Succeeded ||
                        status == Pipelined ||
                        status == Pending </span><span class="cov0" title="0">{
                        occupied += len(tasks)
                }</span>
        }

        <span class="cov0" title="0">return int32(occupied)</span>
}

// Ready returns whether job is ready for run
func (ji *JobInfo) Ready() bool <span class="cov0" title="0">{
        occupied := ji.ReadyTaskNum()

        return occupied &gt;= ji.MinAvailable
}</span>

// IsPending returns whether job is in pending status
func (ji *JobInfo) IsPending() bool <span class="cov0" title="0">{
        if ji.PodGroup == nil || ji.PodGroup.Status.Phase == scheduling.PodGroupPending || ji.PodGroup.Status.Phase == "" </span><span class="cov0" title="0">{
                return true
        }</span>

        <span class="cov0" title="0">return false</span>
}
</pre>
		
		<pre class="file" id="file43" style="display: none">/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "fmt"

        v1 "k8s.io/api/core/v1"
        "k8s.io/client-go/tools/cache"
        "k8s.io/klog"
)

// NamespaceName is name of namespace
type NamespaceName string

const (
        // NamespaceWeightKey is the key in ResourceQuota.spec.hard indicating the weight of this namespace
        NamespaceWeightKey = "volcano.sh/namespace.weight"
        // DefaultNamespaceWeight is the default weight of namespace
        DefaultNamespaceWeight = 1
)

// NamespaceInfo records information of namespace
type NamespaceInfo struct {
        // Name is the name of this namespace
        Name NamespaceName
        // Weight is the highest weight among many ResourceQuota.
        Weight int64
}

// GetWeight returns weight of a namespace, any invalid case would get default value
func (n *NamespaceInfo) GetWeight() int64 <span class="cov0" title="0">{
        if n == nil || n.Weight == 0 </span><span class="cov0" title="0">{
                return DefaultNamespaceWeight
        }</span>
        <span class="cov0" title="0">return n.Weight</span>
}

type quotaItem struct {
        name   string
        weight int64
}

func quotaItemKeyFunc(obj interface{}) (string, error) <span class="cov8" title="1">{
        item, ok := obj.(*quotaItem)
        if !ok </span><span class="cov0" title="0">{
                return "", fmt.Errorf("obj with type %T could not parse", obj)
        }</span>
        <span class="cov8" title="1">return item.name, nil</span>
}

// for big root heap
func quotaItemLessFunc(a interface{}, b interface{}) bool <span class="cov8" title="1">{
        A := a.(*quotaItem)
        B := b.(*quotaItem)
        return A.weight &gt; B.weight
}</span>

// NamespaceCollection will record all details about namespace
type NamespaceCollection struct {
        Name string

        quotaWeight *cache.Heap
}

// NewNamespaceCollection creates new NamespaceCollection object to record all information about a namespace
func NewNamespaceCollection(name string) *NamespaceCollection <span class="cov8" title="1">{
        n := &amp;NamespaceCollection{
                Name:        name,
                quotaWeight: cache.NewHeap(quotaItemKeyFunc, quotaItemLessFunc),
        }
        // add at least one item into quotaWeight.
        // Because cache.Heap.Pop would be blocked until queue is not empty
        n.updateWeight(&amp;quotaItem{
                name:   NamespaceWeightKey,
                weight: DefaultNamespaceWeight,
        })
        return n
}</span>

func (n *NamespaceCollection) deleteWeight(q *quotaItem) <span class="cov8" title="1">{
        n.quotaWeight.Delete(q)
}</span>

func (n *NamespaceCollection) updateWeight(q *quotaItem) <span class="cov8" title="1">{
        n.quotaWeight.Update(q)
}</span>

func itemFromQuota(quota *v1.ResourceQuota) *quotaItem <span class="cov8" title="1">{
        var weight int64 = DefaultNamespaceWeight

        quotaWeight, ok := quota.Spec.Hard[NamespaceWeightKey]
        if ok </span><span class="cov8" title="1">{
                weight = quotaWeight.Value()
        }</span>

        <span class="cov8" title="1">item := &amp;quotaItem{
                name:   quota.Name,
                weight: weight,
        }
        return item</span>
}

// Update modify the registered information according quota object
func (n *NamespaceCollection) Update(quota *v1.ResourceQuota) <span class="cov8" title="1">{
        n.updateWeight(itemFromQuota(quota))
}</span>

// Delete remove the registered information according quota object
func (n *NamespaceCollection) Delete(quota *v1.ResourceQuota) <span class="cov8" title="1">{
        n.deleteWeight(itemFromQuota(quota))
}</span>

// Snapshot will clone a NamespaceInfo without Heap according NamespaceCollection
func (n *NamespaceCollection) Snapshot() *NamespaceInfo <span class="cov8" title="1">{
        var weight int64 = DefaultNamespaceWeight

        obj, err := n.quotaWeight.Pop()
        if err != nil </span><span class="cov0" title="0">{
                klog.Warningf("namespace %s, quota weight meets error %v when pop", n.Name, err)
        }</span> else<span class="cov8" title="1"> {
                item := obj.(*quotaItem)
                weight = item.weight
                n.quotaWeight.Add(item)
        }</span>

        <span class="cov8" title="1">return &amp;NamespaceInfo{
                Name:   NamespaceName(n.Name),
                Weight: weight,
        }</span>
}
</pre>
		
		<pre class="file" id="file44" style="display: none">/*
 Copyright 2021 The Volcano Authors.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package api

import (
        "fmt"
        "strconv"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)

// NodeInfo is node level aggregated information.
type NodeInfo struct {
        Name string
        Node *v1.Node

        // The state of node
        State NodeState

        // The releasing resource on that node
        Releasing *Resource
        // The pipelined resource on that node
        Pipelined *Resource
        // The idle resource on that node
        Idle *Resource
        // The used resource on that node, including running and terminating
        // pods
        Used *Resource

        Allocatable *Resource
        Capability  *Resource

        Tasks             map[TaskID]*TaskInfo
        NumaInfo          *NumatopoInfo
        NumaChgFlag       NumaChgFlag
        NumaSchedulerInfo *NumatopoInfo
        RevocableZone     string

        // Used to store custom information
        Others     map[string]interface{}
        GPUDevices map[int]*GPUDevice

        // enable node resource oversubscription
        OversubscriptionNode bool
        // OfflineJobEvicting true means node resource usage too high then dispatched pod can not use oversubscription resource
        OfflineJobEvicting bool

        // Resource Oversubscription feature: the Oversubscription Resource reported in annotation
        OversubscriptionResource *Resource
}

// FutureIdle returns resources that will be idle in the future:
//
// That is current idle resources plus released resources minus pipelined resources.
func (ni *NodeInfo) FutureIdle() *Resource <span class="cov0" title="0">{
        return ni.Idle.Clone().Add(ni.Releasing).Sub(ni.Pipelined)
}</span>

// GetNodeAllocatable return node Allocatable without OversubscriptionResource resource
func (ni *NodeInfo) GetNodeAllocatable() *Resource <span class="cov0" title="0">{
        return NewResource(ni.Node.Status.Allocatable)
}</span>

// NodeState defines the current state of node.
type NodeState struct {
        Phase  NodePhase
        Reason string
}

// NewNodeInfo is used to create new nodeInfo object
func NewNodeInfo(node *v1.Node) *NodeInfo <span class="cov8" title="1">{
        nodeInfo := &amp;NodeInfo{
                Releasing: EmptyResource(),
                Pipelined: EmptyResource(),
                Idle:      EmptyResource(),
                Used:      EmptyResource(),

                Allocatable: EmptyResource(),
                Capability:  EmptyResource(),

                OversubscriptionResource: EmptyResource(),
                Tasks:                    make(map[TaskID]*TaskInfo),

                GPUDevices: make(map[int]*GPUDevice),
        }

        nodeInfo.setOversubscription(node)

        if node != nil </span><span class="cov8" title="1">{
                nodeInfo.Name = node.Name
                nodeInfo.Node = node
                nodeInfo.Idle = NewResource(node.Status.Allocatable).Add(nodeInfo.OversubscriptionResource)
                nodeInfo.Allocatable = NewResource(node.Status.Allocatable).Add(nodeInfo.OversubscriptionResource)
                nodeInfo.Capability = NewResource(node.Status.Capacity).Add(nodeInfo.OversubscriptionResource)
        }</span>
        <span class="cov8" title="1">nodeInfo.setNodeGPUInfo(node)
        nodeInfo.setNodeState(node)
        nodeInfo.setRevocableZone(node)

        return nodeInfo</span>
}

// RefreshNumaSchedulerInfoByCrd used to update scheduler numa information based the CRD numatopo
func (ni *NodeInfo) RefreshNumaSchedulerInfoByCrd() <span class="cov0" title="0">{
        if ni.NumaInfo == nil </span><span class="cov0" title="0">{
                ni.NumaSchedulerInfo = nil
                return
        }</span>

        <span class="cov0" title="0">tmp := ni.NumaInfo.DeepCopy()
        if ni.NumaChgFlag == NumaInfoMoreFlag </span><span class="cov0" title="0">{
                ni.NumaSchedulerInfo = tmp
        }</span> else<span class="cov0" title="0"> if ni.NumaChgFlag == NumaInfoLessFlag </span><span class="cov0" title="0">{
                numaResMap := ni.NumaSchedulerInfo.NumaResMap
                for resName, resInfo := range tmp.NumaResMap </span><span class="cov0" title="0">{
                        klog.V(5).Infof("resource %s Allocatable : current %v new %v on node %s",
                                resName, numaResMap[resName], resInfo, ni.Name)
                        if numaResMap[resName].Allocatable.Size() &gt;= resInfo.Allocatable.Size() </span><span class="cov0" title="0">{
                                numaResMap[resName].Allocatable = resInfo.Allocatable.Clone()
                                numaResMap[resName].Capacity = resInfo.Capacity
                        }</span>
                }
        }

        <span class="cov0" title="0">ni.NumaChgFlag = NumaInfoResetFlag</span>
}

// Clone used to clone nodeInfo Object
func (ni *NodeInfo) Clone() *NodeInfo <span class="cov0" title="0">{
        res := NewNodeInfo(ni.Node)

        for _, p := range ni.Tasks </span><span class="cov0" title="0">{
                res.AddTask(p)
        }</span>

        <span class="cov0" title="0">if ni.NumaSchedulerInfo != nil </span><span class="cov0" title="0">{
                res.NumaSchedulerInfo = ni.NumaSchedulerInfo.DeepCopy()
                klog.V(5).Infof("node[%s]", ni.Name)
                for resName, resInfo := range res.NumaSchedulerInfo.NumaResMap </span><span class="cov0" title="0">{
                        klog.V(5).Infof("current resource %s : %v", resName, resInfo)
                }</span>

                <span class="cov0" title="0">klog.V(5).Infof("current Policies : %v", res.NumaSchedulerInfo.Policies)</span>
        }

        <span class="cov0" title="0">res.Others = ni.Others
        return res</span>
}

// Ready returns whether node is ready for scheduling
func (ni *NodeInfo) Ready() bool <span class="cov0" title="0">{
        return ni.State.Phase == Ready
}</span>

func (ni *NodeInfo) setRevocableZone(node *v1.Node) <span class="cov8" title="1">{
        if node == nil </span><span class="cov0" title="0">{
                klog.Warningf("the argument node is null.")
                return
        }</span>

        <span class="cov8" title="1">revocableZone := ""
        if len(node.Labels) &gt; 0 </span><span class="cov0" title="0">{
                if value, found := node.Labels[v1beta1.RevocableZone]; found </span><span class="cov0" title="0">{
                        revocableZone = value
                }</span>
        }
        <span class="cov8" title="1">ni.RevocableZone = revocableZone</span>
}

// Check node if enable Oversubscription and set Oversubscription resources
// Only support oversubscription cpu and memory resource for this version
func (ni *NodeInfo) setOversubscription(node *v1.Node) <span class="cov8" title="1">{
        if node == nil </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov8" title="1">ni.OversubscriptionNode = false
        ni.OfflineJobEvicting = false
        if len(node.Labels) &gt; 0 </span><span class="cov0" title="0">{
                if value, found := node.Labels[OversubscriptionNode]; found </span><span class="cov0" title="0">{
                        b, err := strconv.ParseBool(value)
                        if err == nil </span><span class="cov0" title="0">{
                                ni.OversubscriptionNode = b
                        }</span> else<span class="cov0" title="0"> {
                                ni.OversubscriptionNode = false
                        }</span>
                        <span class="cov0" title="0">klog.V(5).Infof("Set node %s Oversubscription to %v", node.Name, ni.OversubscriptionNode)</span>
                }
        }

        <span class="cov8" title="1">if len(node.Annotations) &gt; 0 </span><span class="cov0" title="0">{
                if value, found := node.Annotations[OfflineJobEvicting]; found </span><span class="cov0" title="0">{
                        b, err := strconv.ParseBool(value)
                        if err == nil </span><span class="cov0" title="0">{
                                ni.OfflineJobEvicting = b
                        }</span> else<span class="cov0" title="0"> {
                                ni.OfflineJobEvicting = false
                        }</span>
                        <span class="cov0" title="0">klog.V(5).Infof("Set node %s OfflineJobEvicting to %v", node.Name, ni.OfflineJobEvicting)</span>
                }
                <span class="cov0" title="0">if value, found := node.Annotations[OversubscriptionCPU]; found </span><span class="cov0" title="0">{
                        ni.OversubscriptionResource.MilliCPU, _ = strconv.ParseFloat(value, 64)
                        klog.V(5).Infof("Set node %s Oversubscription CPU to %v", node.Name, ni.OversubscriptionResource.MilliCPU)
                }</span>
                <span class="cov0" title="0">if value, found := node.Annotations[OversubscriptionMemory]; found </span><span class="cov0" title="0">{
                        ni.OversubscriptionResource.Memory, _ = strconv.ParseFloat(value, 64)
                        klog.V(5).Infof("Set node %s Oversubscription Memory to %v", node.Name, ni.OversubscriptionResource.Memory)
                }</span>
        }
}

func (ni *NodeInfo) setNodeState(node *v1.Node) <span class="cov8" title="1">{
        // If node is nil, the node is un-initialized in cache
        if node == nil </span><span class="cov0" title="0">{
                ni.State = NodeState{
                        Phase:  NotReady,
                        Reason: "UnInitialized",
                }
                klog.Warningf("set the node %s status to %s for the reason UnInitialized.", node.Name, NotReady.String())
                return
        }</span>

        // set NodeState according to resources
        <span class="cov8" title="1">if !ni.Used.LessEqual(ni.Allocatable, Zero) </span><span class="cov0" title="0">{
                ni.State = NodeState{
                        Phase:  NotReady,
                        Reason: "OutOfSync",
                }
                return
        }</span>

        // If node not ready, e.g. power off
        <span class="cov8" title="1">for _, cond := range node.Status.Conditions </span><span class="cov0" title="0">{
                if cond.Type == v1.NodeReady &amp;&amp; cond.Status != v1.ConditionTrue </span><span class="cov0" title="0">{
                        ni.State = NodeState{
                                Phase:  NotReady,
                                Reason: "NotReady",
                        }
                        klog.Warningf("set the node %s status to %s.", node.Name, NotReady.String())
                        return
                }</span>
        }

        // Node is ready (ignore node conditions because of taint/toleration)
        <span class="cov8" title="1">ni.State = NodeState{
                Phase:  Ready,
                Reason: "",
        }

        klog.V(4).Infof("set the node %s status to %s.", node.Name, Ready.String())</span>
}

func (ni *NodeInfo) setNodeGPUInfo(node *v1.Node) <span class="cov8" title="1">{
        if node == nil </span><span class="cov0" title="0">{
                return
        }</span>
        <span class="cov8" title="1">memory, ok := node.Status.Capacity[VolcanoGPUResource]
        if !ok </span><span class="cov8" title="1">{
                return
        }</span>
        <span class="cov0" title="0">totalMemory := memory.Value()

        res, ok := node.Status.Capacity[VolcanoGPUNumber]
        if !ok </span><span class="cov0" title="0">{
                return
        }</span>
        <span class="cov0" title="0">gpuNumber := res.Value()
        if gpuNumber == 0 </span><span class="cov0" title="0">{
                klog.Warningf("invalid %s=%s", VolcanoGPUNumber, res.String())
                return
        }</span>

        <span class="cov0" title="0">memoryPerCard := uint(totalMemory / gpuNumber)
        for i := 0; i &lt; int(gpuNumber); i++ </span><span class="cov0" title="0">{
                ni.GPUDevices[i] = NewGPUDevice(i, memoryPerCard)
        }</span>
}

// SetNode sets kubernetes node object to nodeInfo object
func (ni *NodeInfo) SetNode(node *v1.Node) <span class="cov0" title="0">{
        ni.setOversubscription(node)
        ni.setNodeState(node)
        ni.setNodeGPUInfo(node)
        ni.setRevocableZone(node)

        if !ni.Ready() </span><span class="cov0" title="0">{
                klog.Warningf("Failed to set node info, phase: %s, reason: %s",
                        ni.State.Phase, ni.State.Reason)
                return
        }</span>

        <span class="cov0" title="0">ni.Name = node.Name
        ni.Node = node

        ni.Allocatable = NewResource(node.Status.Allocatable).Add(ni.OversubscriptionResource)
        ni.Capability = NewResource(node.Status.Capacity).Add(ni.OversubscriptionResource)
        ni.Releasing = EmptyResource()
        ni.Pipelined = EmptyResource()
        ni.Idle = NewResource(node.Status.Allocatable).Add(ni.OversubscriptionResource)
        ni.Used = EmptyResource()

        for _, ti := range ni.Tasks </span><span class="cov0" title="0">{
                switch ti.Status </span>{
                case Releasing:<span class="cov0" title="0">
                        ni.Idle.Sub(ti.Resreq)
                        ni.Releasing.Add(ti.Resreq)
                        ni.Used.Add(ti.Resreq)
                        ni.AddGPUResource(ti.Pod)</span>
                case Pipelined:<span class="cov0" title="0">
                        ni.Pipelined.Add(ti.Resreq)</span>
                default:<span class="cov0" title="0">
                        ni.Idle.Sub(ti.Resreq)
                        ni.Used.Add(ti.Resreq)
                        ni.AddGPUResource(ti.Pod)</span>
                }
        }
}

func (ni *NodeInfo) allocateIdleResource(ti *TaskInfo) error <span class="cov8" title="1">{
        if ti.Resreq.LessEqual(ni.Idle, Zero) </span><span class="cov8" title="1">{
                ni.Idle.Sub(ti.Resreq)
                return nil
        }</span>

        <span class="cov8" title="1">return fmt.Errorf("selected node NotReady")</span>
}

// AddTask is used to add a task in nodeInfo object
//
// If error occurs both task and node are guaranteed to be in the original state.
func (ni *NodeInfo) AddTask(task *TaskInfo) error <span class="cov8" title="1">{
        if len(task.NodeName) &gt; 0 &amp;&amp; len(ni.Name) &gt; 0 &amp;&amp; task.NodeName != ni.Name </span><span class="cov0" title="0">{
                return fmt.Errorf("task &lt;%v/%v&gt; already on different node &lt;%v&gt;",
                        task.Namespace, task.Name, task.NodeName)
        }</span>

        <span class="cov8" title="1">key := PodKey(task.Pod)
        if _, found := ni.Tasks[key]; found </span><span class="cov0" title="0">{
                return fmt.Errorf("task &lt;%v/%v&gt; already on node &lt;%v&gt;",
                        task.Namespace, task.Name, ni.Name)
        }</span>

        // Node will hold a copy of task to make sure the status
        // change will not impact resource in node.
        <span class="cov8" title="1">ti := task.Clone()

        if ni.Node != nil </span><span class="cov8" title="1">{
                switch ti.Status </span>{
                case Releasing:<span class="cov0" title="0">
                        if err := ni.allocateIdleResource(ti); err != nil </span><span class="cov0" title="0">{
                                return err
                        }</span>
                        <span class="cov0" title="0">ni.Releasing.Add(ti.Resreq)
                        ni.Used.Add(ti.Resreq)
                        ni.AddGPUResource(ti.Pod)</span>
                case Pipelined:<span class="cov0" title="0">
                        ni.Pipelined.Add(ti.Resreq)</span>
                default:<span class="cov8" title="1">
                        if err := ni.allocateIdleResource(ti); err != nil </span><span class="cov8" title="1">{
                                return err
                        }</span>
                        <span class="cov8" title="1">ni.Used.Add(ti.Resreq)
                        ni.AddGPUResource(ti.Pod)</span>
                }
        }

        // Update task node name upon successful task addition.
        <span class="cov8" title="1">task.NodeName = ni.Name
        ti.NodeName = ni.Name
        ni.Tasks[key] = ti

        return nil</span>
}

// RemoveTask used to remove a task from nodeInfo object.
//
// If error occurs both task and node are guaranteed to be in the original state.
func (ni *NodeInfo) RemoveTask(ti *TaskInfo) error <span class="cov8" title="1">{
        key := PodKey(ti.Pod)

        task, found := ni.Tasks[key]
        if !found </span><span class="cov0" title="0">{
                klog.Warningf("failed to find task &lt;%v/%v&gt; on host &lt;%v&gt;",
                        ti.Namespace, ti.Name, ni.Name)
                return nil
        }</span>

        <span class="cov8" title="1">if ni.Node != nil </span><span class="cov8" title="1">{
                switch task.Status </span>{
                case Releasing:<span class="cov0" title="0">
                        ni.Releasing.Sub(task.Resreq)
                        ni.Idle.Add(task.Resreq)
                        ni.Used.Sub(task.Resreq)
                        ni.SubGPUResource(ti.Pod)</span>
                case Pipelined:<span class="cov0" title="0">
                        ni.Pipelined.Sub(task.Resreq)</span>
                default:<span class="cov8" title="1">
                        ni.Idle.Add(task.Resreq)
                        ni.Used.Sub(task.Resreq)
                        ni.SubGPUResource(ti.Pod)</span>
                }
        }

        <span class="cov8" title="1">delete(ni.Tasks, key)

        return nil</span>
}

// UpdateTask is used to update a task in nodeInfo object.
//
// If error occurs both task and node are guaranteed to be in the original state.
func (ni *NodeInfo) UpdateTask(ti *TaskInfo) error <span class="cov0" title="0">{
        if err := ni.RemoveTask(ti); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">if err := ni.AddTask(ti); err != nil </span><span class="cov0" title="0">{
                // This should never happen if task removal was successful,
                // because only possible error during task addition is when task is still on a node.
                klog.Fatalf("Failed to add Task &lt;%s,%s&gt; to Node &lt;%s&gt; during task update",
                        ti.Namespace, ti.Name, ni.Name)
        }</span>
        <span class="cov0" title="0">return nil</span>
}

// String returns nodeInfo details in string format
func (ni NodeInfo) String() string <span class="cov0" title="0">{
        tasks := ""

        i := 0
        for _, task := range ni.Tasks </span><span class="cov0" title="0">{
                tasks += fmt.Sprintf("\n\t %d: %v", i, task)
                i++
        }</span>

        <span class="cov0" title="0">return fmt.Sprintf("Node (%s): allocatable&lt;%v&gt; idle &lt;%v&gt;, used &lt;%v&gt;, releasing &lt;%v&gt;, oversubscribution &lt;%v&gt;, "+
                "state &lt;phase %s, reaseon %s&gt;, oversubscributionNode &lt;%v&gt;, offlineJobEvicting &lt;%v&gt;,taints &lt;%v&gt;%s",
                ni.Name, ni.Allocatable, ni.Idle, ni.Used, ni.Releasing, ni.OversubscriptionResource, ni.State.Phase, ni.State.Reason, ni.OversubscriptionNode, ni.OfflineJobEvicting, ni.Node.Spec.Taints, tasks)</span>
}

// Pods returns all pods running in that node
func (ni *NodeInfo) Pods() (pods []*v1.Pod) <span class="cov0" title="0">{
        for _, t := range ni.Tasks </span><span class="cov0" title="0">{
                pods = append(pods, t.Pod)
        }</span>

        <span class="cov0" title="0">return</span>
}

// GetDevicesIdleGPUMemory returns all the idle GPU memory by gpu card.
func (ni *NodeInfo) GetDevicesIdleGPUMemory() map[int]uint <span class="cov0" title="0">{
        devicesAllGPUMemory := ni.getDevicesAllGPUMemory()
        devicesUsedGPUMemory := ni.getDevicesUsedGPUMemory()
        res := map[int]uint{}
        for id, allMemory := range devicesAllGPUMemory </span><span class="cov0" title="0">{
                if usedMemory, found := devicesUsedGPUMemory[id]; found </span><span class="cov0" title="0">{
                        res[id] = allMemory - usedMemory
                }</span> else<span class="cov0" title="0"> {
                        res[id] = allMemory
                }</span>
        }
        <span class="cov0" title="0">return res</span>
}

func (ni *NodeInfo) getDevicesUsedGPUMemory() map[int]uint <span class="cov0" title="0">{
        res := map[int]uint{}
        for _, device := range ni.GPUDevices </span><span class="cov0" title="0">{
                res[device.ID] = device.getUsedGPUMemory()
        }</span>
        <span class="cov0" title="0">return res</span>
}

func (ni *NodeInfo) getDevicesAllGPUMemory() map[int]uint <span class="cov0" title="0">{
        res := map[int]uint{}
        for _, device := range ni.GPUDevices </span><span class="cov0" title="0">{
                res[device.ID] = device.Memory
        }</span>
        <span class="cov0" title="0">return res</span>
}

// AddGPUResource adds the pod to GPU pool if it is assigned
func (ni *NodeInfo) AddGPUResource(pod *v1.Pod) <span class="cov8" title="1">{
        gpuRes := GetGPUResourceOfPod(pod)
        if gpuRes &gt; 0 </span><span class="cov0" title="0">{
                id := GetGPUIndex(pod)
                if dev := ni.GPUDevices[id]; dev != nil </span><span class="cov0" title="0">{
                        dev.PodMap[string(pod.UID)] = pod
                }</span>
        }
}

// SubGPUResource frees the gpu hold by the pod
func (ni *NodeInfo) SubGPUResource(pod *v1.Pod) <span class="cov8" title="1">{
        gpuRes := GetGPUResourceOfPod(pod)
        if gpuRes &gt; 0 </span><span class="cov0" title="0">{
                id := GetGPUIndex(pod)
                if dev := ni.GPUDevices[id]; dev != nil </span><span class="cov0" title="0">{
                        delete(dev.PodMap, string(pod.UID))
                }</span>
        }
}
</pre>
		
		<pre class="file" id="file45" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        v1 "k8s.io/api/core/v1"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"

        nodeinfov1alpha1 "volcano.sh/apis/pkg/apis/nodeinfo/v1alpha1"
)

// NumaChgFlag indicate node numainfo changed status
type NumaChgFlag int

const (
        // NumaInfoResetFlag indicate reset operate
        NumaInfoResetFlag NumaChgFlag = 0b00
        // NumaInfoMoreFlag indicate the received allocatable resource is getting more
        NumaInfoMoreFlag NumaChgFlag = 0b11
        // NumaInfoLessFlag indicate the received allocatable resource is getting less
        NumaInfoLessFlag NumaChgFlag = 0b10
)

// ResourceInfo is the allocatable information for the resource
type ResourceInfo struct {
        Allocatable cpuset.CPUSet
        Capacity    int
}

// NumatopoInfo is the information about topology manager on the node
type NumatopoInfo struct {
        Namespace   string
        Name        string
        Policies    map[nodeinfov1alpha1.PolicyName]string
        NumaResMap  map[string]*ResourceInfo
        CPUDetail   topology.CPUDetails
        ResReserved v1.ResourceList
}

// DeepCopy used to copy NumatopoInfo
func (info *NumatopoInfo) DeepCopy() *NumatopoInfo <span class="cov0" title="0">{
        numaInfo := &amp;NumatopoInfo{
                Namespace:   info.Namespace,
                Name:        info.Name,
                Policies:    make(map[nodeinfov1alpha1.PolicyName]string),
                NumaResMap:  make(map[string]*ResourceInfo),
                CPUDetail:   topology.CPUDetails{},
                ResReserved: make(v1.ResourceList),
        }

        policies := info.Policies
        for name, policy := range policies </span><span class="cov0" title="0">{
                numaInfo.Policies[name] = policy
        }</span>

        <span class="cov0" title="0">for resName, resInfo := range info.NumaResMap </span><span class="cov0" title="0">{
                var tmpInfo ResourceInfo
                tmpInfo.Capacity = resInfo.Capacity
                tmpInfo.Allocatable = resInfo.Allocatable.Clone()
                numaInfo.NumaResMap[resName] = &amp;tmpInfo
        }</span>

        <span class="cov0" title="0">cpuDetail := info.CPUDetail
        for cpuID, detail := range cpuDetail </span><span class="cov0" title="0">{
                numaInfo.CPUDetail[cpuID] = detail
        }</span>

        <span class="cov0" title="0">resReserved := info.ResReserved
        for resName, res := range resReserved </span><span class="cov0" title="0">{
                numaInfo.ResReserved[resName] = res
        }</span>

        <span class="cov0" title="0">return numaInfo</span>
}

// Compare is the function to show the change of the resource on kubelet
// return val:
// - true : the resource on kubelet is getting more or no change
// - false :  the resource on kubelet is getting less
func (info *NumatopoInfo) Compare(newInfo *NumatopoInfo) bool <span class="cov0" title="0">{
        for resName := range info.NumaResMap </span><span class="cov0" title="0">{
                oldSize := info.NumaResMap[resName].Allocatable.Size()
                newSize := newInfo.NumaResMap[resName].Allocatable.Size()
                if oldSize &lt;= newSize </span><span class="cov0" title="0">{
                        return true
                }</span>
        }

        <span class="cov0" title="0">return false</span>
}

// Allocate is the function to remove the allocated resource
func (info *NumatopoInfo) Allocate(resSets ResNumaSets) <span class="cov0" title="0">{
        for resName := range resSets </span><span class="cov0" title="0">{
                info.NumaResMap[resName].Allocatable = info.NumaResMap[resName].Allocatable.Difference(resSets[resName])
        }</span>
}

// Release is the function to reclaim the allocated resource
func (info *NumatopoInfo) Release(resSets ResNumaSets) <span class="cov0" title="0">{
        for resName := range resSets </span><span class="cov0" title="0">{
                info.NumaResMap[resName].Allocatable = info.NumaResMap[resName].Allocatable.Union(resSets[resName])
        }</span>
}

// GenerateNodeResNumaSets return the idle resource sets of all node
func GenerateNodeResNumaSets(nodes map[string]*NodeInfo) map[string]ResNumaSets <span class="cov0" title="0">{
        nodeSlice := make(map[string]ResNumaSets)
        for _, node := range nodes </span><span class="cov0" title="0">{
                if node.NumaSchedulerInfo == nil </span><span class="cov0" title="0">{
                        continue</span>
                }

                <span class="cov0" title="0">resMaps := make(ResNumaSets)
                for resName, resMap := range node.NumaSchedulerInfo.NumaResMap </span><span class="cov0" title="0">{
                        resMaps[resName] = resMap.Allocatable.Clone()
                }</span>

                <span class="cov0" title="0">nodeSlice[node.Name] = resMaps</span>
        }

        <span class="cov0" title="0">return nodeSlice</span>
}

// GenerateNumaNodes return the numa IDs of all node
func GenerateNumaNodes(nodes map[string]*NodeInfo) map[string][]int <span class="cov0" title="0">{
        nodeNumaMap := make(map[string][]int)

        for _, node := range nodes </span><span class="cov0" title="0">{
                if node.NumaSchedulerInfo == nil </span><span class="cov0" title="0">{
                        continue</span>
                }

                <span class="cov0" title="0">nodeNumaMap[node.Name] = node.NumaSchedulerInfo.CPUDetail.NUMANodes().ToSlice()</span>
        }

        <span class="cov0" title="0">return nodeNumaMap</span>
}

// ResNumaSets is the set map of the resource
type ResNumaSets map[string]cpuset.CPUSet

// Allocate is to remove the allocated resource which is assigned to task
func (resSets ResNumaSets) Allocate(taskSets ResNumaSets) <span class="cov0" title="0">{
        for resName := range taskSets </span><span class="cov0" title="0">{
                if _, ok := resSets[resName]; !ok </span><span class="cov0" title="0">{
                        continue</span>
                }
                <span class="cov0" title="0">resSets[resName] = resSets[resName].Difference(taskSets[resName])</span>
        }
}

// Release is to reclaim the allocated resource which is assigned to task
func (resSets ResNumaSets) Release(taskSets ResNumaSets) <span class="cov0" title="0">{
        for resName := range taskSets </span><span class="cov0" title="0">{
                if _, ok := resSets[resName]; !ok </span><span class="cov0" title="0">{
                        continue</span>
                }
                <span class="cov0" title="0">resSets[resName] = resSets[resName].Union(taskSets[resName])</span>
        }
}

// Clone is the copy action
func (resSets ResNumaSets) Clone() ResNumaSets <span class="cov0" title="0">{
        newSets := make(ResNumaSets)
        for resName := range resSets </span><span class="cov0" title="0">{
                newSets[resName] = resSets[resName].Clone()
        }</span>

        <span class="cov0" title="0">return newSets</span>
}
</pre>
		
		<pre class="file" id="file46" style="display: none">/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "fmt"
        "strconv"
        "strings"
        "time"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)

// Refer k8s.io/kubernetes/pkg/scheduler/algorithm/predicates/predicates.go#GetResourceRequest.
//
// GetResourceRequest returns a *Resource that covers the largest width in each resource dimension.
// Because init-containers run sequentially, we collect the max in each dimension iteratively.
// In contrast, we sum the resource vectors for regular containers since they run simultaneously.
//
// To be consistent with kubernetes default scheduler, it is only used for predicates of actions(e.g.
// allocate, backfill, preempt, reclaim), please use GetPodResourceWithoutInitContainers for other cases.
//
// Example:
//
// Pod:
//   InitContainers
//     IC1:
//       CPU: 2
//       Memory: 1G
//     IC2:
//       CPU: 2
//       Memory: 3G
//   Containers
//     C1:
//       CPU: 2
//       Memory: 1G
//     C2:
//       CPU: 1
//       Memory: 1G
//
// Result: CPU: 3, Memory: 3G

// GetPodResourceRequest returns all the resource required for that pod
func GetPodResourceRequest(pod *v1.Pod) *Resource <span class="cov8" title="1">{
        result := GetPodResourceWithoutInitContainers(pod)

        // take max_resource(sum_pod, any_init_container)
        for _, container := range pod.Spec.InitContainers </span><span class="cov8" title="1">{
                result.SetMaxResource(NewResource(container.Resources.Requests))
        }</span>

        <span class="cov8" title="1">return result</span>
}

// GetPodPreemptable return volcano.sh/preemptable value for pod
func GetPodPreemptable(pod *v1.Pod) bool <span class="cov8" title="1">{
        // check annotaion first
        if len(pod.Annotations) &gt; 0 </span><span class="cov8" title="1">{
                if value, found := pod.Annotations[v1beta1.PodPreemptable]; found </span><span class="cov0" title="0">{
                        b, err := strconv.ParseBool(value)
                        if err != nil </span><span class="cov0" title="0">{
                                klog.Warningf("invalid %s=%s", v1beta1.PodPreemptable, value)
                                return false
                        }</span>
                        <span class="cov0" title="0">return b</span>
                }
        }

        // it annotation does not exit, check label
        <span class="cov8" title="1">if len(pod.Labels) &gt; 0 </span><span class="cov0" title="0">{
                if value, found := pod.Labels[v1beta1.PodPreemptable]; found </span><span class="cov0" title="0">{
                        b, err := strconv.ParseBool(value)
                        if err != nil </span><span class="cov0" title="0">{
                                klog.Warningf("invalid %s=%s", v1beta1.PodPreemptable, value)
                                return false
                        }</span>
                        <span class="cov0" title="0">return b</span>
                }
        }

        <span class="cov8" title="1">return false</span>
}

// GetPodRevocableZone return volcano.sh/revocable-zone value for pod/podgroup
func GetPodRevocableZone(pod *v1.Pod) string <span class="cov8" title="1">{
        if len(pod.Annotations) &gt; 0 </span><span class="cov8" title="1">{
                if value, found := pod.Annotations[v1beta1.RevocableZone]; found </span><span class="cov0" title="0">{
                        if value != "*" </span><span class="cov0" title="0">{
                                return ""
                        }</span>
                        <span class="cov0" title="0">return value</span>
                }

                <span class="cov8" title="1">if value, found := pod.Annotations[v1beta1.PodPreemptable]; found </span><span class="cov0" title="0">{
                        if b, err := strconv.ParseBool(value); err == nil &amp;&amp; b </span><span class="cov0" title="0">{
                                return "*"
                        }</span>
                }
        }
        <span class="cov8" title="1">return ""</span>
}

// GetPodTopologyPolicy return volcano.sh/numa-topology-policy value for pod
func GetPodTopologyPolicy(pod *v1.Pod) string <span class="cov8" title="1">{
        if len(pod.Annotations) &gt; 0 </span><span class="cov8" title="1">{
                if value, found := pod.Annotations[v1beta1.NumaPolicyKey]; found </span><span class="cov0" title="0">{
                        return value
                }</span>
        }
        <span class="cov8" title="1">return ""</span>
}

// GetPodResourceWithoutInitContainers returns Pod's resource request, it does not contain
// init containers' resource request.
func GetPodResourceWithoutInitContainers(pod *v1.Pod) *Resource <span class="cov8" title="1">{
        result := EmptyResource()
        for _, container := range pod.Spec.Containers </span><span class="cov8" title="1">{
                result.Add(NewResource(container.Resources.Requests))
        }</span>

        <span class="cov8" title="1">return result</span>
}

// GetGPUIndex returns the ID of the GPU
func GetGPUIndex(pod *v1.Pod) int <span class="cov0" title="0">{
        if len(pod.Annotations) &gt; 0 </span><span class="cov0" title="0">{
                value, found := pod.Annotations[GPUIndex]
                if found </span><span class="cov0" title="0">{
                        id, err := strconv.Atoi(value)
                        if err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("invalid %s=%s", GPUIndex, value)
                                return -1
                        }</span>
                        <span class="cov0" title="0">return id</span>
                }
        }

        <span class="cov0" title="0">return -1</span>
}

func escapeJSONPointer(p string) string <span class="cov0" title="0">{
        // Escaping reference name using https://tools.ietf.org/html/rfc6901
        p = strings.Replace(p, "~", "~0", -1)
        p = strings.Replace(p, "/", "~1", -1)
        return p
}</span>

// AddGPUIndexPatch returns the patch adding GPU index
func AddGPUIndexPatch(id int) string <span class="cov0" title="0">{
        return fmt.Sprintf(`[{"op": "add", "path": "/metadata/annotations/%s", "value":"%d"},`+
                `{"op": "add", "path": "/metadata/annotations/%s", "value": "%d"}]`,
                escapeJSONPointer(PredicateTime), time.Now().UnixNano(),
                escapeJSONPointer(GPUIndex), id)
}</span>

// RemoveGPUIndexPatch returns the patch removing GPU index
func RemoveGPUIndexPatch() string <span class="cov0" title="0">{
        return fmt.Sprintf(`[{"op": "remove", "path": "/metadata/annotations/%s"},`+
                `{"op": "remove", "path": "/metadata/annotations/%s"]`, escapeJSONPointer(PredicateTime), escapeJSONPointer(GPUIndex))
}</span>
</pre>
		
		<pre class="file" id="file47" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "k8s.io/apimachinery/pkg/types"

        "volcano.sh/apis/pkg/apis/scheduling"
        "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)

// QueueID is UID type, serves as unique ID for each queue
type QueueID types.UID

// QueueInfo will have all details about queue
type QueueInfo struct {
        UID  QueueID
        Name string

        Weight int32

        // Weights is a list of slash sperated float numbers.
        // Each of them is a weight corresponding the
        // hierarchy level.
        Weights string
        // Hierarchy is a list of node name along the
        // path from the root to the node itself.
        Hierarchy string

        Queue *scheduling.Queue
}

// NewQueueInfo creates new queueInfo object
func NewQueueInfo(queue *scheduling.Queue) *QueueInfo <span class="cov0" title="0">{
        return &amp;QueueInfo{
                UID:  QueueID(queue.Name),
                Name: queue.Name,

                Weight:    queue.Spec.Weight,
                Hierarchy: queue.Annotations[v1beta1.KubeHierarchyAnnotationKey],
                Weights:   queue.Annotations[v1beta1.KubeHierarchyWeightAnnotationKey],

                Queue: queue,
        }
}</span>

// Clone is used to clone queueInfo object
func (q *QueueInfo) Clone() *QueueInfo <span class="cov0" title="0">{
        return &amp;QueueInfo{
                UID:       q.UID,
                Name:      q.Name,
                Weight:    q.Weight,
                Hierarchy: q.Hierarchy,
                Weights:   q.Weights,
                Queue:     q.Queue,
        }
}</span>

// Reclaimable return whether queue is reclaimable
func (q *QueueInfo) Reclaimable() bool <span class="cov0" title="0">{
        if q == nil </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov0" title="0">if q.Queue == nil </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov0" title="0">if q.Queue.Spec.Reclaimable == nil </span><span class="cov0" title="0">{
                return true
        }</span>

        <span class="cov0" title="0">return *q.Queue.Spec.Reclaimable</span>
}
</pre>
		
		<pre class="file" id="file48" style="display: none">/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "fmt"
        "math"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/resource"
        v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"

        "volcano.sh/volcano/pkg/scheduler/util/assert"
)

const (
        // GPUResourceName need to follow https://github.com/NVIDIA/k8s-device-plugin/blob/66a35b71ac4b5cbfb04714678b548bd77e5ba719/server.go#L20
        GPUResourceName = "nvidia.com/gpu"
)

const (
        minResource float64 = 0.1
)

// DimensionDefaultValue means default value for black resource dimension
type DimensionDefaultValue string

const (
        // Zero means resource dimension not defined will be treated as zero
        Zero DimensionDefaultValue = "Zero"
        // Infinity means resource dimension not defined will be treated as infinity
        Infinity DimensionDefaultValue = "Infinity"
)

// Resource struct defines all the resource type
type Resource struct {
        MilliCPU float64
        Memory   float64

        // ScalarResources
        ScalarResources map[v1.ResourceName]float64

        // MaxTaskNum is only used by predicates; it should NOT
        // be accounted in other operators, e.g. Add.
        MaxTaskNum int
}

// EmptyResource creates a empty resource object and returns
func EmptyResource() *Resource <span class="cov8" title="1">{
        return &amp;Resource{}
}</span>

// NewResource creates a new resource object from resource list
func NewResource(rl v1.ResourceList) *Resource <span class="cov8" title="1">{
        r := EmptyResource()
        for rName, rQuant := range rl </span><span class="cov8" title="1">{
                switch rName </span>{
                case v1.ResourceCPU:<span class="cov8" title="1">
                        r.MilliCPU += float64(rQuant.MilliValue())</span>
                case v1.ResourceMemory:<span class="cov8" title="1">
                        r.Memory += float64(rQuant.Value())</span>
                case v1.ResourcePods:<span class="cov0" title="0">
                        r.MaxTaskNum += int(rQuant.Value())</span>
                default:<span class="cov8" title="1">
                        //NOTE: When converting this back to k8s resource, we need record the format as well as / 1000
                        if v1helper.IsScalarResourceName(rName) </span><span class="cov8" title="1">{
                                r.AddScalar(rName, float64(rQuant.MilliValue()))
                        }</span>
                }
        }
        <span class="cov8" title="1">return r</span>
}

// Clone is used to clone a resource type, which is a deep copy function.
func (r *Resource) Clone() *Resource <span class="cov8" title="1">{
        clone := &amp;Resource{
                MilliCPU:   r.MilliCPU,
                Memory:     r.Memory,
                MaxTaskNum: r.MaxTaskNum,
        }

        if r.ScalarResources != nil </span><span class="cov8" title="1">{
                clone.ScalarResources = make(map[v1.ResourceName]float64)
                for k, v := range r.ScalarResources </span><span class="cov8" title="1">{
                        clone.ScalarResources[k] = v
                }</span>
        }

        <span class="cov8" title="1">return clone</span>
}

// String returns resource details in string format
func (r *Resource) String() string <span class="cov0" title="0">{
        str := fmt.Sprintf("cpu %0.2f, memory %0.2f", r.MilliCPU, r.Memory)
        for rName, rQuant := range r.ScalarResources </span><span class="cov0" title="0">{
                str = fmt.Sprintf("%s, %s %0.2f", str, rName, rQuant)
        }</span>
        <span class="cov0" title="0">return str</span>
}

// ResourceNames returns all resource types
func (r *Resource) ResourceNames() ResourceNameList <span class="cov0" title="0">{
        resNames := ResourceNameList{}

        if r.MilliCPU &gt;= minResource </span><span class="cov0" title="0">{
                resNames = append(resNames, v1.ResourceCPU)
        }</span>

        <span class="cov0" title="0">if r.Memory &gt;= minResource </span><span class="cov0" title="0">{
                resNames = append(resNames, v1.ResourceMemory)
        }</span>

        <span class="cov0" title="0">for rName, rMount := range r.ScalarResources </span><span class="cov0" title="0">{
                if rMount &gt;= minResource </span><span class="cov0" title="0">{
                        resNames = append(resNames, rName)
                }</span>
        }

        <span class="cov0" title="0">return resNames</span>
}

// Get returns the resource value for that particular resource type
func (r *Resource) Get(rn v1.ResourceName) float64 <span class="cov0" title="0">{
        switch rn </span>{
        case v1.ResourceCPU:<span class="cov0" title="0">
                return r.MilliCPU</span>
        case v1.ResourceMemory:<span class="cov0" title="0">
                return r.Memory</span>
        default:<span class="cov0" title="0">
                if r.ScalarResources == nil </span><span class="cov0" title="0">{
                        return 0
                }</span>
                <span class="cov0" title="0">return r.ScalarResources[rn]</span>
        }
}

// IsEmpty returns false if any kind of resource is not less than min value, otherwise returns true
func (r *Resource) IsEmpty() bool <span class="cov8" title="1">{
        if !(r.MilliCPU &lt; minResource &amp;&amp; r.Memory &lt; minResource) </span><span class="cov8" title="1">{
                return false
        }</span>

        <span class="cov0" title="0">for _, rQuant := range r.ScalarResources </span><span class="cov0" title="0">{
                if rQuant &gt;= minResource </span><span class="cov0" title="0">{
                        return false
                }</span>
        }

        <span class="cov0" title="0">return true</span>
}

// IsZero returns false if the given kind of resource is not less than min value
func (r *Resource) IsZero(rn v1.ResourceName) bool <span class="cov8" title="1">{
        switch rn </span>{
        case v1.ResourceCPU:<span class="cov8" title="1">
                return r.MilliCPU &lt; minResource</span>
        case v1.ResourceMemory:<span class="cov0" title="0">
                return r.Memory &lt; minResource</span>
        default:<span class="cov8" title="1">
                if r.ScalarResources == nil </span><span class="cov0" title="0">{
                        return true
                }</span>

                <span class="cov8" title="1">_, found := r.ScalarResources[rn]
                assert.Assertf(found, "unknown resource %s", rn)

                return r.ScalarResources[rn] &lt; minResource</span>
        }
}

// Add is used to add two given resources
func (r *Resource) Add(rr *Resource) *Resource <span class="cov8" title="1">{
        r.MilliCPU += rr.MilliCPU
        r.Memory += rr.Memory

        for rName, rQuant := range rr.ScalarResources </span><span class="cov8" title="1">{
                if r.ScalarResources == nil </span><span class="cov8" title="1">{
                        r.ScalarResources = map[v1.ResourceName]float64{}
                }</span>
                <span class="cov8" title="1">r.ScalarResources[rName] += rQuant</span>
        }

        <span class="cov8" title="1">return r</span>
}

//Sub subtracts two Resource objects.
func (r *Resource) Sub(rr *Resource) *Resource <span class="cov8" title="1">{
        assert.Assertf(rr.LessEqual(r, Zero), "resource is not sufficient to do operation: &lt;%v&gt; sub &lt;%v&gt;", r, rr)

        r.MilliCPU -= rr.MilliCPU
        r.Memory -= rr.Memory

        if r.ScalarResources == nil </span><span class="cov8" title="1">{
                return r
        }</span>
        <span class="cov8" title="1">for rrName, rrQuant := range rr.ScalarResources </span><span class="cov8" title="1">{
                r.ScalarResources[rrName] -= rrQuant
        }</span>

        <span class="cov8" title="1">return r</span>
}

// Multi multiples the resource with ratio provided
func (r *Resource) Multi(ratio float64) *Resource <span class="cov0" title="0">{
        r.MilliCPU *= ratio
        r.Memory *= ratio
        for rName, rQuant := range r.ScalarResources </span><span class="cov0" title="0">{
                r.ScalarResources[rName] = rQuant * ratio
        }</span>
        <span class="cov0" title="0">return r</span>
}

// SetMaxResource compares with ResourceList and takes max value for each Resource.
func (r *Resource) SetMaxResource(rr *Resource) <span class="cov8" title="1">{
        if r == nil || rr == nil </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov8" title="1">if rr.MilliCPU &gt; r.MilliCPU </span><span class="cov8" title="1">{
                r.MilliCPU = rr.MilliCPU
        }</span>
        <span class="cov8" title="1">if rr.Memory &gt; r.Memory </span><span class="cov8" title="1">{
                r.Memory = rr.Memory
        }</span>

        <span class="cov8" title="1">for rrName, rrQuant := range rr.ScalarResources </span><span class="cov8" title="1">{
                if r.ScalarResources == nil </span><span class="cov8" title="1">{
                        r.ScalarResources = make(map[v1.ResourceName]float64)
                        for k, v := range rr.ScalarResources </span><span class="cov8" title="1">{
                                r.ScalarResources[k] = v
                        }</span>
                        <span class="cov8" title="1">return</span>
                }
                <span class="cov8" title="1">_, ok := r.ScalarResources[rrName]
                if !ok || rrQuant &gt; r.ScalarResources[rrName] </span><span class="cov8" title="1">{
                        r.ScalarResources[rrName] = rrQuant
                }</span>
        }
}

//FitDelta Computes the delta between a resource object representing available
//resources an operand representing resources being requested.  Any
//field that is less than 0 after the operation represents an
//insufficient resource.
func (r *Resource) FitDelta(rr *Resource) *Resource <span class="cov0" title="0">{
        if rr.MilliCPU &gt; 0 </span><span class="cov0" title="0">{
                r.MilliCPU -= rr.MilliCPU + minResource
        }</span>

        <span class="cov0" title="0">if rr.Memory &gt; 0 </span><span class="cov0" title="0">{
                r.Memory -= rr.Memory + minResource
        }</span>

        <span class="cov0" title="0">if r.ScalarResources == nil </span><span class="cov0" title="0">{
                r.ScalarResources = make(map[v1.ResourceName]float64)
        }</span>

        <span class="cov0" title="0">for rrName, rrQuant := range rr.ScalarResources </span><span class="cov0" title="0">{
                if rrQuant &gt; 0 </span><span class="cov0" title="0">{
                        _, ok := r.ScalarResources[rrName]
                        if !ok </span><span class="cov0" title="0">{
                                r.ScalarResources[rrName] = 0
                        }</span>
                        <span class="cov0" title="0">r.ScalarResources[rrName] -= rrQuant + minResource</span>
                }
        }

        <span class="cov0" title="0">return r</span>
}

// Less returns true only on condition that all dimensions of resources in r are less than that of rr,
// Otherwise returns false.
// @param defaultValue "default value for resource dimension not defined in ScalarResources. Its value can only be one of 'Zero' and 'Infinity'"
func (r *Resource) Less(rr *Resource, defaultValue DimensionDefaultValue) bool <span class="cov8" title="1">{
        lessFunc := func(l, r float64) bool </span><span class="cov8" title="1">{
                return l &lt; r
        }</span>

        <span class="cov8" title="1">leftResource := r.Clone()
        rightResource := rr.Clone()

        if !lessFunc(leftResource.MilliCPU, rightResource.MilliCPU) </span><span class="cov8" title="1">{
                return false
        }</span>
        <span class="cov8" title="1">if !lessFunc(leftResource.Memory, rightResource.Memory) </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov8" title="1">r.setDefaultValue(leftResource, rightResource, defaultValue)

        for resourceName, leftValue := range leftResource.ScalarResources </span><span class="cov8" title="1">{
                rightValue := rightResource.ScalarResources[resourceName]
                if rightValue == -1 </span><span class="cov8" title="1">{
                        continue</span>
                }
                <span class="cov8" title="1">if leftValue == -1 || !lessFunc(leftValue, rightValue) </span><span class="cov8" title="1">{
                        return false
                }</span>
        }
        <span class="cov8" title="1">return true</span>
}

// LessEqual returns true only on condition that all dimensions of resources in r are less than or equal with that of rr,
// Otherwise returns false.
// @param defaultValue "default value for resource dimension not defined in ScalarResources. Its value can only be one of 'Zero' and 'Infinity'"
func (r *Resource) LessEqual(rr *Resource, defaultValue DimensionDefaultValue) bool <span class="cov8" title="1">{
        lessEqualFunc := func(l, r, diff float64) bool </span><span class="cov8" title="1">{
                if l &lt; r || math.Abs(l-r) &lt; diff </span><span class="cov8" title="1">{
                        return true
                }</span>
                <span class="cov8" title="1">return false</span>
        }

        <span class="cov8" title="1">leftResource := r.Clone()
        rightResource := rr.Clone()

        if !lessEqualFunc(leftResource.MilliCPU, rightResource.MilliCPU, minResource) </span><span class="cov8" title="1">{
                return false
        }</span>
        <span class="cov8" title="1">if !lessEqualFunc(leftResource.Memory, rightResource.Memory, minResource) </span><span class="cov8" title="1">{
                return false
        }</span>

        <span class="cov8" title="1">r.setDefaultValue(leftResource, rightResource, defaultValue)

        for resourceName, leftValue := range leftResource.ScalarResources </span><span class="cov8" title="1">{
                rightValue := rightResource.ScalarResources[resourceName]
                if rightValue == -1 </span><span class="cov0" title="0">{
                        continue</span>
                }
                <span class="cov8" title="1">if leftValue == -1 || !lessEqualFunc(leftValue, rightValue, minResource) </span><span class="cov8" title="1">{
                        return false
                }</span>
        }
        <span class="cov8" title="1">return true</span>
}

// LessPartly returns true if there exists any dimension whose resource amount in r is less than that in rr.
// Otherwise returns false.
// @param defaultValue "default value for resource dimension not defined in ScalarResources. Its value can only be one of 'Zero' and 'Infinity'"
func (r *Resource) LessPartly(rr *Resource, defaultValue DimensionDefaultValue) bool <span class="cov8" title="1">{
        lessFunc := func(l, r float64) bool </span><span class="cov8" title="1">{
                return l &lt; r
        }</span>

        <span class="cov8" title="1">leftResource := r.Clone()
        rightResource := rr.Clone()

        if lessFunc(leftResource.MilliCPU, rightResource.MilliCPU) || lessFunc(leftResource.Memory, rightResource.Memory) </span><span class="cov8" title="1">{
                return true
        }</span>

        <span class="cov8" title="1">r.setDefaultValue(leftResource, rightResource, defaultValue)

        for resourceName, leftValue := range leftResource.ScalarResources </span><span class="cov8" title="1">{
                rightValue := rightResource.ScalarResources[resourceName]
                if leftValue == -1 </span><span class="cov8" title="1">{
                        continue</span>
                }
                <span class="cov8" title="1">if rightValue == -1 || lessFunc(leftValue, rightValue) </span><span class="cov8" title="1">{
                        return true
                }</span>
        }
        <span class="cov8" title="1">return false</span>
}

// LessEqualPartly returns true if there exists any dimension whose resource amount in r is less than or equal with that in rr.
// Otherwise returns false.
// @param defaultValue "default value for resource dimension not defined in ScalarResources. Its value can only be one of 'Zero' and 'Infinity'"
func (r *Resource) LessEqualPartly(rr *Resource, defaultValue DimensionDefaultValue) bool <span class="cov8" title="1">{
        lessEqualFunc := func(l, r, diff float64) bool </span><span class="cov8" title="1">{
                if l &lt; r || math.Abs(l-r) &lt; diff </span><span class="cov8" title="1">{
                        return true
                }</span>
                <span class="cov8" title="1">return false</span>
        }

        <span class="cov8" title="1">leftResource := r.Clone()
        rightResource := rr.Clone()

        if lessEqualFunc(leftResource.MilliCPU, rightResource.MilliCPU, minResource) || lessEqualFunc(leftResource.Memory, rightResource.Memory, minResource) </span><span class="cov8" title="1">{
                return true
        }</span>

        <span class="cov8" title="1">r.setDefaultValue(leftResource, rightResource, defaultValue)

        for resourceName, leftValue := range leftResource.ScalarResources </span><span class="cov8" title="1">{
                rightValue := rightResource.ScalarResources[resourceName]
                if leftValue == -1 </span><span class="cov0" title="0">{
                        continue</span>
                }
                <span class="cov8" title="1">if rightValue == -1 || lessEqualFunc(leftValue, rightValue, minResource) </span><span class="cov8" title="1">{
                        return true
                }</span>
        }
        <span class="cov8" title="1">return false</span>
}

// Equal returns true only on condition that values in all dimension are equal with each other for r and rr
// Otherwise returns false.
// @param defaultValue "default value for resource dimension not defined in ScalarResources. Its value can only be one of 'Zero' and 'Infinity'"
func (r *Resource) Equal(rr *Resource, defaultValue DimensionDefaultValue) bool <span class="cov8" title="1">{
        equalFunc := func(l, r, diff float64) bool </span><span class="cov8" title="1">{
                return l == r || math.Abs(l-r) &lt; diff
        }</span>

        <span class="cov8" title="1">leftResource := r.Clone()
        rightResource := rr.Clone()

        if !equalFunc(leftResource.MilliCPU, rightResource.MilliCPU, minResource) || !equalFunc(leftResource.Memory, rightResource.Memory, minResource) </span><span class="cov8" title="1">{
                return false
        }</span>

        <span class="cov8" title="1">r.setDefaultValue(leftResource, rightResource, defaultValue)

        for resourceName, leftValue := range leftResource.ScalarResources </span><span class="cov8" title="1">{
                rightValue := rightResource.ScalarResources[resourceName]
                if !equalFunc(leftValue, rightValue, minResource) </span><span class="cov0" title="0">{
                        return false
                }</span>
        }
        <span class="cov8" title="1">return true</span>
}

// Diff calculate the difference between two resource object
// Note: if `defaultValue` equals `Infinity`, the difference between two values will be `Infinity`, marked as -1
func (r *Resource) Diff(rr *Resource, defaultValue DimensionDefaultValue) (*Resource, *Resource) <span class="cov8" title="1">{
        leftRes := r.Clone()
        rightRes := rr.Clone()
        increasedVal := EmptyResource()
        decreasedVal := EmptyResource()
        r.setDefaultValue(leftRes, rightRes, defaultValue)

        if leftRes.MilliCPU &gt; rightRes.MilliCPU </span><span class="cov8" title="1">{
                increasedVal.MilliCPU = leftRes.MilliCPU - rightRes.MilliCPU
        }</span> else<span class="cov8" title="1"> {
                decreasedVal.MilliCPU = rightRes.MilliCPU - leftRes.MilliCPU
        }</span>

        <span class="cov8" title="1">if leftRes.Memory &gt; rightRes.Memory </span><span class="cov8" title="1">{
                increasedVal.Memory = leftRes.Memory - rightRes.Memory
        }</span> else<span class="cov8" title="1"> {
                decreasedVal.Memory = rightRes.Memory - leftRes.Memory
        }</span>

        <span class="cov8" title="1">increasedVal.ScalarResources = make(map[v1.ResourceName]float64, 0)
        decreasedVal.ScalarResources = make(map[v1.ResourceName]float64, 0)
        for lName, lQuant := range leftRes.ScalarResources </span><span class="cov8" title="1">{
                rQuant, _ := rightRes.ScalarResources[lName]
                if lQuant == -1 </span><span class="cov8" title="1">{
                        increasedVal.ScalarResources[lName] = -1
                        continue</span>
                }
                <span class="cov8" title="1">if rQuant == -1 </span><span class="cov8" title="1">{
                        decreasedVal.ScalarResources[lName] = -1
                        continue</span>
                }
                <span class="cov8" title="1">if lQuant &gt; rQuant </span><span class="cov8" title="1">{
                        increasedVal.ScalarResources[lName] = lQuant - rQuant
                }</span> else<span class="cov8" title="1"> {
                        decreasedVal.ScalarResources[lName] = rQuant - lQuant
                }</span>
        }

        <span class="cov8" title="1">return increasedVal, decreasedVal</span>
}

// AddScalar adds a resource by a scalar value of this resource.
func (r *Resource) AddScalar(name v1.ResourceName, quantity float64) <span class="cov8" title="1">{
        r.SetScalar(name, r.ScalarResources[name]+quantity)
}</span>

// SetScalar sets a resource by a scalar value of this resource.
func (r *Resource) SetScalar(name v1.ResourceName, quantity float64) <span class="cov8" title="1">{
        // Lazily allocate scalar resource map.
        if r.ScalarResources == nil </span><span class="cov8" title="1">{
                r.ScalarResources = map[v1.ResourceName]float64{}
        }</span>
        <span class="cov8" title="1">r.ScalarResources[name] = quantity</span>
}

// MinDimensionResource is used to reset the r resource dimension which is less than rr
// e.g r resource is &lt;cpu 2000.00, memory 4047845376.00, hugepages-2Mi 0.00, hugepages-1Gi 0.00&gt;
// rr resource is &lt;cpu 3000.00, memory 1000.00&gt;
// return r resource is &lt;cpu 2000.00, memory 1000.00, hugepages-2Mi 0.00, hugepages-1Gi 0.00&gt;
func (r *Resource) MinDimensionResource(rr *Resource) *Resource <span class="cov8" title="1">{
        if rr.MilliCPU &lt; r.MilliCPU </span><span class="cov8" title="1">{
                r.MilliCPU = rr.MilliCPU
        }</span>
        <span class="cov8" title="1">if rr.Memory &lt; r.Memory </span><span class="cov8" title="1">{
                r.Memory = rr.Memory
        }</span>

        <span class="cov8" title="1">if rr.ScalarResources == nil </span><span class="cov0" title="0">{
                if r.ScalarResources != nil </span><span class="cov0" title="0">{
                        for name := range r.ScalarResources </span><span class="cov0" title="0">{
                                r.ScalarResources[name] = 0
                        }</span>
                }
        } else<span class="cov8" title="1"> {
                if r.ScalarResources != nil </span><span class="cov8" title="1">{
                        for name, quant := range rr.ScalarResources </span><span class="cov8" title="1">{
                                if quant &lt; r.ScalarResources[name] </span><span class="cov8" title="1">{
                                        r.ScalarResources[name] = quant
                                }</span>
                        }
                }
        }
        <span class="cov8" title="1">return r</span>
}

// setDefaultValue sets default value for resource dimension not defined of ScalarResource in leftResource and rightResource
// @param defaultValue "default value for resource dimension not defined in ScalarResources. It can only be one of 'Zero' or 'Infinity'"
func (r *Resource) setDefaultValue(leftResource, rightResource *Resource, defaultValue DimensionDefaultValue) <span class="cov8" title="1">{
        if leftResource.ScalarResources == nil </span><span class="cov8" title="1">{
                leftResource.ScalarResources = map[v1.ResourceName]float64{}
        }</span>
        <span class="cov8" title="1">if rightResource.ScalarResources == nil </span><span class="cov8" title="1">{
                rightResource.ScalarResources = map[v1.ResourceName]float64{}
        }</span>
        <span class="cov8" title="1">for resourceName := range leftResource.ScalarResources </span><span class="cov8" title="1">{
                _, ok := rightResource.ScalarResources[resourceName]
                if !ok </span><span class="cov8" title="1">{
                        if defaultValue == Zero </span><span class="cov8" title="1">{
                                rightResource.ScalarResources[resourceName] = 0
                        }</span> else<span class="cov8" title="1"> if defaultValue == Infinity </span><span class="cov8" title="1">{
                                rightResource.ScalarResources[resourceName] = -1
                        }</span>
                }
        }

        <span class="cov8" title="1">for resourceName := range rightResource.ScalarResources </span><span class="cov8" title="1">{
                _, ok := leftResource.ScalarResources[resourceName]
                if !ok </span><span class="cov8" title="1">{
                        if defaultValue == Zero </span><span class="cov8" title="1">{
                                leftResource.ScalarResources[resourceName] = 0
                        }</span> else<span class="cov8" title="1"> if defaultValue == Infinity </span><span class="cov8" title="1">{
                                leftResource.ScalarResources[resourceName] = -1
                        }</span>
                }
        }
}

// ParseResourceList parses the given configuration map into an API
// ResourceList or returns an error.
func ParseResourceList(m map[string]string) (v1.ResourceList, error) <span class="cov0" title="0">{
        if len(m) == 0 </span><span class="cov0" title="0">{
                return nil, nil
        }</span>
        <span class="cov0" title="0">rl := make(v1.ResourceList)
        for k, v := range m </span><span class="cov0" title="0">{
                switch v1.ResourceName(k) </span>{
                // CPU, memory, local storage, and PID resources are supported.
                case v1.ResourceCPU, v1.ResourceMemory, v1.ResourceEphemeralStorage:<span class="cov0" title="0">
                        q, err := resource.ParseQuantity(v)
                        if err != nil </span><span class="cov0" title="0">{
                                return nil, err
                        }</span>
                        <span class="cov0" title="0">if q.Sign() == -1 </span><span class="cov0" title="0">{
                                return nil, fmt.Errorf("resource quantity for %q cannot be negative: %v", k, v)
                        }</span>
                        <span class="cov0" title="0">rl[v1.ResourceName(k)] = q</span>
                default:<span class="cov0" title="0">
                        return nil, fmt.Errorf("cannot reserve %q resource", k)</span>
                }
        }
        <span class="cov0" title="0">return rl, nil</span>
}

func GetMinResource() float64 <span class="cov0" title="0">{
        return minResource
}</span>

// ResourceNameList struct defines resource name collection
type ResourceNameList []v1.ResourceName

// Contains judges whether rr is subset of r
func (r ResourceNameList) Contains(rr ResourceNameList) bool <span class="cov0" title="0">{
        for _, rrName := range ([]v1.ResourceName)(rr) </span><span class="cov0" title="0">{
                isResourceExist := false
                for _, rName := range ([]v1.ResourceName)(r) </span><span class="cov0" title="0">{
                        if rName == rrName </span><span class="cov0" title="0">{
                                isResourceExist = true
                                break</span>
                        }
                }
                <span class="cov0" title="0">if !isResourceExist </span><span class="cov0" title="0">{
                        return false
                }</span>
        }
        <span class="cov0" title="0">return true</span>
}
</pre>
		
		<pre class="file" id="file49" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "k8s.io/apimachinery/pkg/types"

        "volcano.sh/apis/pkg/apis/scheduling"
)

// ClusterID is UID type, serves as unique ID for each queue
type ClusterID types.UID

// SiloClusterInfo will have all details about queue
type SiloClusterInfo struct {
        UID     ClusterID
        Cluster *scheduling.Cluster
}

// NewSiloClusterInfo creates new queueInfo object
func NewSiloClusterInfo(cluster *scheduling.Cluster) *SiloClusterInfo <span class="cov0" title="0">{
        return &amp;SiloClusterInfo{
                UID:     ClusterID(cluster.Name),
                Cluster: cluster,
        }
}</span>
</pre>
		
		<pre class="file" id="file50" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "fmt"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/resource"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
)

func buildNode(name string, alloc v1.ResourceList) *v1.Node <span class="cov8" title="1">{
        return &amp;v1.Node{
                ObjectMeta: metav1.ObjectMeta{
                        Name: name,
                },
                Status: v1.NodeStatus{
                        Capacity:    alloc,
                        Allocatable: alloc,
                },
        }
}</span>

func buildPod(ns, n, nn string, p v1.PodPhase, req v1.ResourceList, owner []metav1.OwnerReference, labels map[string]string) *v1.Pod <span class="cov8" title="1">{
        return &amp;v1.Pod{
                ObjectMeta: metav1.ObjectMeta{
                        UID:             types.UID(fmt.Sprintf("%v-%v", ns, n)),
                        Name:            n,
                        Namespace:       ns,
                        OwnerReferences: owner,
                        Labels:          labels,
                },
                Status: v1.PodStatus{
                        Phase: p,
                },
                Spec: v1.PodSpec{
                        NodeName: nn,
                        Containers: []v1.Container{
                                {
                                        Resources: v1.ResourceRequirements{
                                                Requests: req,
                                        },
                                },
                        },
                },
        }
}</span>

func buildResourceList(cpu string, memory string) v1.ResourceList <span class="cov8" title="1">{
        return v1.ResourceList{
                v1.ResourceCPU:    resource.MustParse(cpu),
                v1.ResourceMemory: resource.MustParse(memory),
        }
}</span>

func buildResource(cpu string, memory string) *Resource <span class="cov8" title="1">{
        return NewResource(v1.ResourceList{
                v1.ResourceCPU:    resource.MustParse(cpu),
                v1.ResourceMemory: resource.MustParse(memory),
        })
}</span>

func buildOwnerReference(owner string) metav1.OwnerReference <span class="cov8" title="1">{
        controller := true
        return metav1.OwnerReference{
                Controller: &amp;controller,
                UID:        types.UID(owner),
        }
}</span>
</pre>
		
		<pre class="file" id="file51" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        k8sframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
)

// TaskStatus defines the status of a task/pod.
type TaskStatus int

const (
        // Pending means the task is pending in the apiserver.
        Pending TaskStatus = 1 &lt;&lt; iota

        // Allocated means the scheduler assigns a host to it.
        Allocated

        // Pipelined means the scheduler assigns a host to wait for releasing resource.
        Pipelined

        // Binding means the scheduler send Bind request to apiserver.
        Binding

        // Bound means the task/Pod bounds to a host.
        Bound

        // Running means a task is running on the host.
        Running

        // Releasing means a task/pod is deleted.
        Releasing

        // Succeeded means that all containers in the pod have voluntarily terminated
        // with a container exit code of 0, and the system is not going to restart any of these containers.
        Succeeded

        // Failed means that all containers in the pod have terminated, and at least one container has
        // terminated in a failure (exited with a non-zero exit code or was stopped by the system).
        Failed

        // Unknown means the status of task/pod is unknown to the scheduler.
        Unknown
)

func (ts TaskStatus) String() string <span class="cov8" title="1">{
        switch ts </span>{
        case Pending:<span class="cov8" title="1">
                return "Pending"</span>
        case Allocated:<span class="cov0" title="0">
                return "Allocated"</span>
        case Pipelined:<span class="cov0" title="0">
                return "Pipelined"</span>
        case Binding:<span class="cov0" title="0">
                return "Binding"</span>
        case Bound:<span class="cov0" title="0">
                return "Bound"</span>
        case Running:<span class="cov0" title="0">
                return "Running"</span>
        case Releasing:<span class="cov0" title="0">
                return "Releasing"</span>
        case Succeeded:<span class="cov0" title="0">
                return "Succeeded"</span>
        case Failed:<span class="cov0" title="0">
                return "Failed"</span>
        default:<span class="cov0" title="0">
                return "Unknown"</span>
        }
}

// NodePhase defines the phase of node
type NodePhase int

const (
        // Ready means the node is ready for scheduling
        Ready NodePhase = 1 &lt;&lt; iota
        // NotReady means the node is not ready for scheduling
        NotReady
)

func (np NodePhase) String() string <span class="cov8" title="1">{
        switch np </span>{
        case Ready:<span class="cov8" title="1">
                return "Ready"</span>
        case NotReady:<span class="cov0" title="0">
                return "NotReady"</span>
        }

        <span class="cov0" title="0">return "Unknown"</span>
}

// validateStatusUpdate validates whether the status transfer is valid.
func validateStatusUpdate(oldStatus, newStatus TaskStatus) error <span class="cov0" title="0">{
        return nil
}</span>

// LessFn is the func declaration used by sort or priority queue.
type LessFn func(interface{}, interface{}) bool

// CompareFn is the func declaration used by sort or priority queue.
type CompareFn func(interface{}, interface{}) int

// ValidateFn is the func declaration used to check object's status.
type ValidateFn func(interface{}) bool

// ValidateResult is struct to which can used to determine the result
type ValidateResult struct {
        Pass    bool
        Reason  string
        Message string
}

// ValidateExFn is the func declaration used to validate the result.
type ValidateExFn func(interface{}) *ValidateResult

// VoteFn is the func declaration used to check object's complicated status.
type VoteFn func(interface{}) int

// JobEnqueuedFn is the func declaration used to call after job enqueued.
type JobEnqueuedFn func(interface{})

// PredicateFn is the func declaration used to predicate node for task.
type PredicateFn func(*TaskInfo, *NodeInfo) error

// BestNodeFn is the func declaration used to return the nodeScores to plugins.
type BestNodeFn func(*TaskInfo, map[float64][]*NodeInfo) *NodeInfo

// EvictableFn is the func declaration used to evict tasks.
type EvictableFn func(*TaskInfo, []*TaskInfo) ([]*TaskInfo, int)

// NodeOrderFn is the func declaration used to get priority score for a node for a particular task.
type NodeOrderFn func(*TaskInfo, *NodeInfo) (float64, error)

// BatchNodeOrderFn is the func declaration used to get priority score for ALL nodes for a particular task.
type BatchNodeOrderFn func(*TaskInfo, []*NodeInfo) (map[string]float64, error)

// NodeMapFn is the func declaration used to get priority score for a node for a particular task.
type NodeMapFn func(*TaskInfo, *NodeInfo) (float64, error)

// NodeReduceFn is the func declaration used to reduce priority score for a node for a particular task.
type NodeReduceFn func(*TaskInfo, k8sframework.NodeScoreList) error

// NodeOrderMapFn is the func declaration used to get priority score of all plugins for a node for a particular task.
type NodeOrderMapFn func(*TaskInfo, *NodeInfo) (map[string]float64, float64, error)

// NodeOrderReduceFn is the func declaration used to reduce priority score of all nodes for a plugin for a particular task.
type NodeOrderReduceFn func(*TaskInfo, map[string]k8sframework.NodeScoreList) (map[string]float64, error)

// TargetJobFn is the func declaration used to select the target job satisfies some conditions
type TargetJobFn func([]*JobInfo) *JobInfo

// ReservedNodesFn is the func declaration used to select the reserved nodes
type ReservedNodesFn func()

// VictimTasksFn is the func declaration used to select victim tasks
type VictimTasksFn func() []*TaskInfo

// UnderUsedResourceFn is the func declaration used to get under used resource list for queue
type UnderUsedResourceFn func(*QueueInfo) ResourceNameList
</pre>
		
		<pre class="file" id="file52" style="display: none">package api

import (
        "fmt"
        "sort"
        "strings"
)

const (
        // NodePodNumberExceeded means pods in node exceed the allocatable pod number
        NodePodNumberExceeded = "node(s) pod number exceeded"
        // NodeResourceFitFailed means node could not fit the request of pod
        NodeResourceFitFailed = "node(s) resource fit failed"

        // AllNodeUnavailableMsg is the default error message
        AllNodeUnavailableMsg = "all nodes are unavailable"
)

// These are reasons for a pod's transition to a condition.
const (
        // PodReasonUnschedulable reason in PodScheduled PodCondition means that the scheduler
        // can't schedule the pod right now, for example due to insufficient resources in the cluster.
        PodReasonUnschedulable = "Unschedulable"
        // PodReasonSchedulable reason in PodScheduled PodCondition means that the scheduler
        // can schedule the pod right now, but not bind yet
        PodReasonSchedulable = "Schedulable"
        // PodReasonUndetermined reason in PodScheduled PodCondition means that the scheduler
        // skips scheduling the pod which left the pod `Undetermined`, for example due to unschedulable pod already occurred.
        PodReasonUndetermined = "Undetermined"
)

// FitErrors is set of FitError on many nodes
type FitErrors struct {
        nodes map[string]*FitError
        err   string
}

// NewFitErrors returns an FitErrors
func NewFitErrors() *FitErrors <span class="cov0" title="0">{
        f := new(FitErrors)
        f.nodes = make(map[string]*FitError)
        return f
}</span>

// SetError set the common error message in FitErrors
func (f *FitErrors) SetError(err string) <span class="cov0" title="0">{
        f.err = err
}</span>

// SetNodeError set the node error in FitErrors
func (f *FitErrors) SetNodeError(nodeName string, err error) <span class="cov0" title="0">{
        var fe *FitError
        switch obj := err.(type) </span>{
        case *FitError:<span class="cov0" title="0">
                obj.NodeName = nodeName
                fe = obj</span>
        default:<span class="cov0" title="0">
                fe = &amp;FitError{
                        NodeName: nodeName,
                        Reasons:  []string{obj.Error()},
                }</span>
        }

        <span class="cov0" title="0">f.nodes[nodeName] = fe</span>
}

// Error returns the final error message
func (f *FitErrors) Error() string <span class="cov8" title="1">{
        reasons := make(map[string]int)

        for _, node := range f.nodes </span><span class="cov8" title="1">{
                for _, reason := range node.Reasons </span><span class="cov8" title="1">{
                        reasons[reason]++
                }</span>
        }

        <span class="cov8" title="1">sortReasonsHistogram := func() []string </span><span class="cov8" title="1">{
                reasonStrings := []string{}
                for k, v := range reasons </span><span class="cov8" title="1">{
                        reasonStrings = append(reasonStrings, fmt.Sprintf("%v %v", v, k))
                }</span>
                <span class="cov8" title="1">sort.Strings(reasonStrings)
                return reasonStrings</span>
        }
        <span class="cov8" title="1">if f.err == "" </span><span class="cov8" title="1">{
                f.err = AllNodeUnavailableMsg
        }</span>
        <span class="cov8" title="1">reasonMsg := fmt.Sprintf(f.err+": %v.", strings.Join(sortReasonsHistogram(), ", "))
        return reasonMsg</span>
}

// FitError describe the reason why task could not fit that node
type FitError struct {
        taskNamespace string
        taskName      string
        NodeName      string
        Reasons       []string
}

// NewFitError return FitError by message
func NewFitError(task *TaskInfo, node *NodeInfo, message ...string) *FitError <span class="cov0" title="0">{
        fe := &amp;FitError{
                taskName:      task.Name,
                taskNamespace: task.Namespace,
                NodeName:      node.Name,
                Reasons:       message,
        }
        return fe
}</span>

// Error returns the final error message
func (f *FitError) Error() string <span class="cov0" title="0">{
        return fmt.Sprintf("task %s/%s on node %s fit failed: %s", f.taskNamespace, f.taskName, f.NodeName, strings.Join(f.Reasons, ", "))
}</span>
</pre>
		
		<pre class="file" id="file53" style="display: none">/*
 Copyright 2021 The Volcano Authors.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package cache

import (
        "context"
        "fmt"
        "sync"
        "time"

        v1 "k8s.io/api/core/v1"
        "k8s.io/api/scheduling/v1beta1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/runtime"
        utilruntime "k8s.io/apimachinery/pkg/util/runtime"
        "k8s.io/apimachinery/pkg/util/wait"
        "k8s.io/client-go/informers"
        infov1 "k8s.io/client-go/informers/core/v1"
        schedv1 "k8s.io/client-go/informers/scheduling/v1beta1"
        storagev1 "k8s.io/client-go/informers/storage/v1"
        storagev1alpha1 "k8s.io/client-go/informers/storage/v1alpha1"
        "k8s.io/client-go/kubernetes"
        corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
        "k8s.io/client-go/rest"
        "k8s.io/client-go/tools/cache"
        "k8s.io/client-go/tools/record"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"
        podutil "k8s.io/kubernetes/pkg/api/v1/pod"
        volumescheduling "k8s.io/kubernetes/pkg/controller/volume/scheduling"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/apis/scheduling"
        schedulingscheme "volcano.sh/apis/pkg/apis/scheduling/scheme"
        vcv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        vcclient "volcano.sh/apis/pkg/client/clientset/versioned"
        "volcano.sh/apis/pkg/client/clientset/versioned/scheme"
        vcinformer "volcano.sh/apis/pkg/client/informers/externalversions"
        cpuinformerv1 "volcano.sh/apis/pkg/client/informers/externalversions/nodeinfo/v1alpha1"
        vcinformerv1 "volcano.sh/apis/pkg/client/informers/externalversions/scheduling/v1beta1"

        "volcano.sh/volcano/cmd/scheduler/app/options"
        schedulingapi "volcano.sh/volcano/pkg/scheduler/api"
)

func init() <span class="cov8" title="1">{
        schemeBuilder := runtime.SchemeBuilder{
                v1.AddToScheme,
        }

        utilruntime.Must(schemeBuilder.AddToScheme(scheme.Scheme))
}</span>

// New returns a Cache implementation.
func New(config *rest.Config, schedulerName string, defaultQueue string) Cache <span class="cov0" title="0">{
        return newSchedulerCache(config, schedulerName, defaultQueue)
}</span>

// SchedulerCache cache for the kube batch
type SchedulerCache struct {
        sync.Mutex

        kubeClient   *kubernetes.Clientset
        vcClient     *vcclient.Clientset
        defaultQueue string
        // schedulerName is the name for volcano scheduler
        schedulerName string

        podInformer                infov1.PodInformer
        nodeInformer               infov1.NodeInformer
        podGroupInformerV1beta1    vcinformerv1.PodGroupInformer
        queueInformerV1beta1       vcinformerv1.QueueInformer
        pvInformer                 infov1.PersistentVolumeInformer
        pvcInformer                infov1.PersistentVolumeClaimInformer
        scInformer                 storagev1.StorageClassInformer
        pcInformer                 schedv1.PriorityClassInformer
        quotaInformer              infov1.ResourceQuotaInformer
        csiNodeInformer            storagev1.CSINodeInformer
        csiDriverInformer          storagev1.CSIDriverInformer
        csiStorageCapacityInformer storagev1alpha1.CSIStorageCapacityInformer
        cpuInformer                cpuinformerv1.NumatopologyInformer

        Binder         Binder
        Evictor        Evictor
        StatusUpdater  StatusUpdater
        PodGroupBinder BatchBinder
        VolumeBinder   VolumeBinder

        Recorder record.EventRecorder

        Jobs                 map[schedulingapi.JobID]*schedulingapi.JobInfo
        Nodes                map[string]*schedulingapi.NodeInfo
        Queues               map[schedulingapi.QueueID]*schedulingapi.QueueInfo
        PriorityClasses      map[string]*v1beta1.PriorityClass
        NodeList             []string
        defaultPriorityClass *v1beta1.PriorityClass
        defaultPriority      int32

        NamespaceCollection map[string]*schedulingapi.NamespaceCollection

        errTasks    workqueue.RateLimitingInterface
        deletedJobs workqueue.RateLimitingInterface

        informerFactory informers.SharedInformerFactory
}

type defaultBinder struct {
        kubeclient *kubernetes.Clientset
}

//Bind will send bind request to api server
func (db *defaultBinder) Bind(p *v1.Pod, hostname string) error <span class="cov0" title="0">{
        if err := db.kubeclient.CoreV1().Pods(p.Namespace).Bind(context.TODO(),
                &amp;v1.Binding{
                        ObjectMeta: metav1.ObjectMeta{Namespace: p.Namespace, Name: p.Name, UID: p.UID, Annotations: p.Annotations},
                        Target: v1.ObjectReference{
                                Kind: "Node",
                                Name: hostname,
                        },
                },
                metav1.CreateOptions{}); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to bind pod &lt;%v/%v&gt;: %#v", p.Namespace, p.Name, err)
                return err
        }</span>
        <span class="cov0" title="0">return nil</span>
}

type defaultEvictor struct {
        kubeclient *kubernetes.Clientset
        recorder   record.EventRecorder
}

// Evict will send delete pod request to api server
func (de *defaultEvictor) Evict(p *v1.Pod, reason string) error <span class="cov0" title="0">{
        klog.V(3).Infof("Evicting pod %v/%v, because of %v", p.Namespace, p.Name, reason)

        evictMsg := fmt.Sprintf("Pod is evicted, because of %v", reason)
        annotations := map[string]string{}
        // record that we are evicting the pod
        de.recorder.AnnotatedEventf(p, annotations, v1.EventTypeWarning, "Evict", evictMsg)

        pod := p.DeepCopy()
        condition := &amp;v1.PodCondition{
                Type:    v1.PodReady,
                Status:  v1.ConditionFalse,
                Reason:  "Evict",
                Message: evictMsg,
        }
        if !podutil.UpdatePodCondition(&amp;pod.Status, condition) </span><span class="cov0" title="0">{
                klog.V(1).Infof("UpdatePodCondition: existed condition, not update")
                klog.V(1).Infof("%+v", pod.Status.Conditions)
                return nil
        }</span>
        <span class="cov0" title="0">if _, err := de.kubeclient.CoreV1().Pods(p.Namespace).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{}); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to update pod &lt;%v/%v&gt; status: %v", pod.Namespace, pod.Name, err)
                return err
        }</span>
        <span class="cov0" title="0">if err := de.kubeclient.CoreV1().Pods(p.Namespace).Delete(context.TODO(), p.Name, metav1.DeleteOptions{}); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to evict pod &lt;%v/%v&gt;: %#v", p.Namespace, p.Name, err)
                return err
        }</span>

        <span class="cov0" title="0">return nil</span>
}

// defaultStatusUpdater is the default implementation of the StatusUpdater interface
type defaultStatusUpdater struct {
        kubeclient *kubernetes.Clientset
        vcclient   *vcclient.Clientset
}

// following the same logic as podutil.UpdatePodCondition
func podConditionHaveUpdate(status *v1.PodStatus, condition *v1.PodCondition) bool <span class="cov0" title="0">{
        lastTransitionTime := metav1.Now()
        // Try to find this pod condition.
        _, oldCondition := podutil.GetPodCondition(status, condition.Type)

        if oldCondition == nil </span><span class="cov0" title="0">{
                // We are adding new pod condition.
                return true
        }</span>
        // We are updating an existing condition, so we need to check if it has changed.
        <span class="cov0" title="0">if condition.Status == oldCondition.Status </span><span class="cov0" title="0">{
                lastTransitionTime = oldCondition.LastTransitionTime
        }</span>

        <span class="cov0" title="0">isEqual := condition.Status == oldCondition.Status &amp;&amp;
                condition.Reason == oldCondition.Reason &amp;&amp;
                condition.Message == oldCondition.Message &amp;&amp;
                condition.LastProbeTime.Equal(&amp;oldCondition.LastProbeTime) &amp;&amp;
                lastTransitionTime.Equal(&amp;oldCondition.LastTransitionTime)

        // Return true if one of the fields have changed.
        return !isEqual</span>
}

// UpdatePodCondition will Update pod with podCondition
func (su *defaultStatusUpdater) UpdatePodCondition(pod *v1.Pod, condition *v1.PodCondition) (*v1.Pod, error) <span class="cov0" title="0">{
        klog.V(3).Infof("Updating pod condition for %s/%s to (%s==%s)", pod.Namespace, pod.Name, condition.Type, condition.Status)
        if podutil.UpdatePodCondition(&amp;pod.Status, condition) </span><span class="cov0" title="0">{
                return su.kubeclient.CoreV1().Pods(pod.Namespace).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{})
        }</span>
        <span class="cov0" title="0">return pod, nil</span>
}

// UpdatePodGroup will Update pod with podCondition
func (su *defaultStatusUpdater) UpdatePodGroup(pg *schedulingapi.PodGroup) (*schedulingapi.PodGroup, error) <span class="cov0" title="0">{
        podgroup := &amp;vcv1beta1.PodGroup{}
        if err := schedulingscheme.Scheme.Convert(&amp;pg.PodGroup, podgroup, nil); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err)
                return nil, err
        }</span>

        <span class="cov0" title="0">updated, err := su.vcclient.SchedulingV1beta1().PodGroups(podgroup.Namespace).Update(context.TODO(), podgroup, metav1.UpdateOptions{})
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Error while updating PodGroup with error: %v", err)
                return nil, err
        }</span>

        <span class="cov0" title="0">podGroupInfo := &amp;schedulingapi.PodGroup{Version: schedulingapi.PodGroupVersionV1Beta1}
        if err := schedulingscheme.Scheme.Convert(updated, &amp;podGroupInfo.PodGroup, nil); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Error while converting v1alpha.PodGroup to api.PodGroup with error: %v", err)
                return nil, err
        }</span>

        <span class="cov0" title="0">return podGroupInfo, nil</span>
}

type defaultVolumeBinder struct {
        volumeBinder volumescheduling.SchedulerVolumeBinder
}

// AllocateVolumes allocates volume on the host to the task
func (dvb *defaultVolumeBinder) AllocateVolumes(task *schedulingapi.TaskInfo, hostname string, podVolumes *volumescheduling.PodVolumes) error <span class="cov0" title="0">{
        allBound, err := dvb.volumeBinder.AssumePodVolumes(task.Pod, hostname, podVolumes)
        task.VolumeReady = allBound

        return err
}</span>

// GetPodVolumes get pod volume on the host
func (dvb *defaultVolumeBinder) GetPodVolumes(task *schedulingapi.TaskInfo,
        node *v1.Node) (podVolumes *volumescheduling.PodVolumes, err error) <span class="cov0" title="0">{
        boundClaims, claimsToBind, _, err := dvb.volumeBinder.GetPodVolumes(task.Pod)
        if err != nil </span><span class="cov0" title="0">{
                return nil, err
        }</span>

        <span class="cov0" title="0">podVolumes, _, err = dvb.volumeBinder.FindPodVolumes(task.Pod, boundClaims, claimsToBind, node)
        return podVolumes, err</span>
}

// BindVolumes binds volumes to the task
func (dvb *defaultVolumeBinder) BindVolumes(task *schedulingapi.TaskInfo, podVolumes *volumescheduling.PodVolumes) error <span class="cov0" title="0">{
        // If task's volumes are ready, did not bind them again.
        if task.VolumeReady </span><span class="cov0" title="0">{
                return nil
        }</span>

        <span class="cov0" title="0">return dvb.volumeBinder.BindPodVolumes(task.Pod, podVolumes)</span>
}

type podgroupBinder struct {
        kubeclient *kubernetes.Clientset
        vcclient   *vcclient.Clientset
}

// Bind will add silo cluster annotaion on pod and podgroup
func (pgb *podgroupBinder) Bind(job *schedulingapi.JobInfo, cluster string) (*schedulingapi.JobInfo, error) <span class="cov0" title="0">{
        if len(job.Tasks) == 0 </span><span class="cov0" title="0">{
                klog.V(4).Infof("Job pods have not been created yet")
                return job, nil
        }</span>
        <span class="cov0" title="0">for _, task := range job.Tasks </span><span class="cov0" title="0">{
                pod := task.Pod
                pod.Annotations[batch.ForwardClusterKey] = cluster
                pod.ResourceVersion = ""
                _, err := pgb.kubeclient.CoreV1().Pods(pod.Namespace).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{})
                if err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Error while update pod annotation with error: %v", err)
                        return nil, err
                }</span>
        }

        <span class="cov0" title="0">pg := job.PodGroup
        pg.Annotations[batch.ForwardClusterKey] = cluster
        podgroup := &amp;vcv1beta1.PodGroup{}
        if err := schedulingscheme.Scheme.Convert(&amp;pg.PodGroup, podgroup, nil); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err)
                return nil, err
        }</span>
        <span class="cov0" title="0">newPg, err := pgb.vcclient.SchedulingV1beta1().PodGroups(pg.Namespace).Update(context.TODO(), podgroup, metav1.UpdateOptions{})
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Error while update PodGroup annotation with error: %v", err)
                return nil, err
        }</span>
        <span class="cov0" title="0">job.PodGroup.ResourceVersion = newPg.ResourceVersion
        klog.V(4).Infof("Bind PodGroup &lt;%s&gt; successfully", job.PodGroup.Name)
        return job, nil</span>
}

func newSchedulerCache(config *rest.Config, schedulerName string, defaultQueue string) *SchedulerCache <span class="cov0" title="0">{
        kubeClient, err := kubernetes.NewForConfig(config)
        if err != nil </span><span class="cov0" title="0">{
                panic(fmt.Sprintf("failed init kubeClient, with err: %v", err))</span>
        }
        <span class="cov0" title="0">vcClient, err := vcclient.NewForConfig(config)
        if err != nil </span><span class="cov0" title="0">{
                panic(fmt.Sprintf("failed init vcClient, with err: %v", err))</span>
        }
        <span class="cov0" title="0">eventClient, err := kubernetes.NewForConfig(config)
        if err != nil </span><span class="cov0" title="0">{
                panic(fmt.Sprintf("failed init eventClient, with err: %v", err))</span>
        }

        // create default queue
        <span class="cov0" title="0">reclaimable := true
        defaultQue := vcv1beta1.Queue{
                ObjectMeta: metav1.ObjectMeta{
                        Name: defaultQueue,
                },
                Spec: vcv1beta1.QueueSpec{
                        Reclaimable: &amp;reclaimable,
                        Weight:      1,
                },
        }
        if _, err := vcClient.SchedulingV1beta1().Queues().Create(context.TODO(), &amp;defaultQue, metav1.CreateOptions{}); err != nil &amp;&amp; !apierrors.IsAlreadyExists(err) </span><span class="cov0" title="0">{
                panic(fmt.Sprintf("failed init default queue, with err: %v", err))</span>
        }

        <span class="cov0" title="0">sc := &amp;SchedulerCache{
                Jobs:            make(map[schedulingapi.JobID]*schedulingapi.JobInfo),
                Nodes:           make(map[string]*schedulingapi.NodeInfo),
                Queues:          make(map[schedulingapi.QueueID]*schedulingapi.QueueInfo),
                PriorityClasses: make(map[string]*v1beta1.PriorityClass),
                errTasks:        workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
                deletedJobs:     workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
                kubeClient:      kubeClient,
                vcClient:        vcClient,
                defaultQueue:    defaultQueue,
                schedulerName:   schedulerName,

                NamespaceCollection: make(map[string]*schedulingapi.NamespaceCollection),

                NodeList: []string{},
        }

        // Prepare event clients.
        broadcaster := record.NewBroadcaster()
        broadcaster.StartRecordingToSink(&amp;corev1.EventSinkImpl{Interface: eventClient.CoreV1().Events("")})
        sc.Recorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: schedulerName})

        sc.Binder = &amp;defaultBinder{
                kubeclient: sc.kubeClient,
        }

        sc.Evictor = &amp;defaultEvictor{
                kubeclient: sc.kubeClient,
                recorder:   sc.Recorder,
        }

        sc.StatusUpdater = &amp;defaultStatusUpdater{
                kubeclient: sc.kubeClient,
                vcclient:   sc.vcClient,
        }

        sc.PodGroupBinder = &amp;podgroupBinder{
                kubeclient: sc.kubeClient,
                vcclient:   sc.vcClient,
        }

        informerFactory := informers.NewSharedInformerFactory(sc.kubeClient, 0)
        sc.informerFactory = informerFactory

        // create informer for node information
        sc.nodeInformer = informerFactory.Core().V1().Nodes()
        sc.nodeInformer.Informer().AddEventHandlerWithResyncPeriod(
                cache.ResourceEventHandlerFuncs{
                        AddFunc:    sc.AddNode,
                        UpdateFunc: sc.UpdateNode,
                        DeleteFunc: sc.DeleteNode,
                },
                0,
        )

        sc.podInformer = informerFactory.Core().V1().Pods()
        sc.pvcInformer = informerFactory.Core().V1().PersistentVolumeClaims()
        sc.pvInformer = informerFactory.Core().V1().PersistentVolumes()
        sc.scInformer = informerFactory.Storage().V1().StorageClasses()
        sc.csiNodeInformer = informerFactory.Storage().V1().CSINodes()
        sc.csiDriverInformer = informerFactory.Storage().V1().CSIDrivers()
        sc.csiStorageCapacityInformer = informerFactory.Storage().V1alpha1().CSIStorageCapacities()
        sc.VolumeBinder = &amp;defaultVolumeBinder{
                volumeBinder: volumescheduling.NewVolumeBinder(
                        sc.kubeClient,
                        sc.podInformer,
                        sc.nodeInformer,
                        sc.csiNodeInformer,
                        sc.pvcInformer,
                        sc.pvInformer,
                        sc.scInformer,
                        &amp;volumescheduling.CapacityCheck{
                                CSIDriverInformer:          sc.csiDriverInformer,
                                CSIStorageCapacityInformer: sc.csiStorageCapacityInformer,
                        },
                        30*time.Second,
                ),
        }

        // create informer for pod information
        sc.podInformer.Informer().AddEventHandler(
                cache.FilteringResourceEventHandler{
                        FilterFunc: func(obj interface{}) bool </span><span class="cov0" title="0">{
                                switch v := obj.(type) </span>{
                                case *v1.Pod:<span class="cov0" title="0">
                                        if !responsibleForPod(v, schedulerName) </span><span class="cov0" title="0">{
                                                if len(v.Spec.NodeName) == 0 </span><span class="cov0" title="0">{
                                                        return false
                                                }</span>
                                        }
                                        <span class="cov0" title="0">return true</span>
                                default:<span class="cov0" title="0">
                                        return false</span>
                                }
                        },
                        Handler: cache.ResourceEventHandlerFuncs{
                                AddFunc:    sc.AddPod,
                                UpdateFunc: sc.UpdatePod,
                                DeleteFunc: sc.DeletePod,
                        },
                })

        <span class="cov0" title="0">sc.pcInformer = informerFactory.Scheduling().V1beta1().PriorityClasses()
        sc.pcInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    sc.AddPriorityClass,
                UpdateFunc: sc.UpdatePriorityClass,
                DeleteFunc: sc.DeletePriorityClass,
        })

        sc.quotaInformer = informerFactory.Core().V1().ResourceQuotas()
        sc.quotaInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    sc.AddResourceQuota,
                UpdateFunc: sc.UpdateResourceQuota,
                DeleteFunc: sc.DeleteResourceQuota,
        })

        vcinformers := vcinformer.NewSharedInformerFactory(sc.vcClient, 0)

        // create informer for PodGroup(v1beta1) information
        sc.podGroupInformerV1beta1 = vcinformers.Scheduling().V1beta1().PodGroups()
        sc.podGroupInformerV1beta1.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    sc.AddPodGroupV1beta1,
                UpdateFunc: sc.UpdatePodGroupV1beta1,
                DeleteFunc: sc.DeletePodGroupV1beta1,
        })

        // create informer(v1beta1) for Queue information
        sc.queueInformerV1beta1 = vcinformers.Scheduling().V1beta1().Queues()
        sc.queueInformerV1beta1.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    sc.AddQueueV1beta1,
                UpdateFunc: sc.UpdateQueueV1beta1,
                DeleteFunc: sc.DeleteQueueV1beta1,
        })

        sc.cpuInformer = vcinformers.Nodeinfo().V1alpha1().Numatopologies()
        sc.cpuInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    sc.AddNumaInfoV1alpha1,
                UpdateFunc: sc.UpdateNumaInfoV1alpha1,
                DeleteFunc: sc.DeleteNumaInfoV1alpha1,
        })
        return sc</span>
}

// Run  starts the schedulerCache
func (sc *SchedulerCache) Run(stopCh &lt;-chan struct{}) <span class="cov0" title="0">{
        go sc.podInformer.Informer().Run(stopCh)
        go sc.nodeInformer.Informer().Run(stopCh)
        go sc.podGroupInformerV1beta1.Informer().Run(stopCh)
        go sc.pvInformer.Informer().Run(stopCh)
        go sc.pvcInformer.Informer().Run(stopCh)
        go sc.scInformer.Informer().Run(stopCh)
        go sc.queueInformerV1beta1.Informer().Run(stopCh)
        go sc.quotaInformer.Informer().Run(stopCh)
        go sc.cpuInformer.Informer().Run(stopCh)

        if options.ServerOpts.EnablePriorityClass </span><span class="cov0" title="0">{
                go sc.pcInformer.Informer().Run(stopCh)
        }</span>

        // Re-sync error tasks.
        <span class="cov0" title="0">go wait.Until(sc.processResyncTask, 0, stopCh)

        // Cleanup jobs.
        go wait.Until(sc.processCleanupJob, 0, stopCh)</span>
}

// WaitForCacheSync sync the cache with the api server
func (sc *SchedulerCache) WaitForCacheSync(stopCh &lt;-chan struct{}) bool <span class="cov0" title="0">{
        return cache.WaitForCacheSync(stopCh,
                func() []cache.InformerSynced </span><span class="cov0" title="0">{
                        informerSynced := []cache.InformerSynced{
                                sc.podInformer.Informer().HasSynced,
                                sc.podGroupInformerV1beta1.Informer().HasSynced,
                                sc.nodeInformer.Informer().HasSynced,
                                sc.pvInformer.Informer().HasSynced,
                                sc.pvcInformer.Informer().HasSynced,
                                sc.scInformer.Informer().HasSynced,
                                sc.queueInformerV1beta1.Informer().HasSynced,
                                sc.quotaInformer.Informer().HasSynced,
                                sc.cpuInformer.Informer().HasSynced,
                        }
                        if options.ServerOpts.EnablePriorityClass </span><span class="cov0" title="0">{
                                informerSynced = append(informerSynced, sc.pcInformer.Informer().HasSynced)
                        }</span>
                        <span class="cov0" title="0">return informerSynced</span>
                }()...,
        )
}

// findJobAndTask returns job and the task info
func (sc *SchedulerCache) findJobAndTask(taskInfo *schedulingapi.TaskInfo) (*schedulingapi.JobInfo, *schedulingapi.TaskInfo, error) <span class="cov8" title="1">{
        job, found := sc.Jobs[taskInfo.Job]
        if !found </span><span class="cov0" title="0">{
                return nil, nil, fmt.Errorf("failed to find Job %v for Task %v",
                        taskInfo.Job, taskInfo.UID)
        }</span>

        <span class="cov8" title="1">task, found := job.Tasks[taskInfo.UID]
        if !found </span><span class="cov0" title="0">{
                return nil, nil, fmt.Errorf("failed to find task in status %v by id %v",
                        taskInfo.Status, taskInfo.UID)
        }</span>

        <span class="cov8" title="1">return job, task, nil</span>
}

// Evict will evict the pod.
//
// If error occurs both task and job are guaranteed to be in the original state.
func (sc *SchedulerCache) Evict(taskInfo *schedulingapi.TaskInfo, reason string) error <span class="cov0" title="0">{
        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        job, task, err := sc.findJobAndTask(taskInfo)

        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">node, found := sc.Nodes[task.NodeName]
        if !found </span><span class="cov0" title="0">{
                return fmt.Errorf("failed to bind Task %v to host %v, host does not exist",
                        task.UID, task.NodeName)
        }</span>

        <span class="cov0" title="0">originalStatus := task.Status
        if err := job.UpdateTaskStatus(task, schedulingapi.Releasing); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        // Add new task to node.
        <span class="cov0" title="0">if err := node.UpdateTask(task); err != nil </span><span class="cov0" title="0">{
                // After failing to update task to a node we need to revert task status from Releasing,
                // otherwise task might be stuck in the Releasing state indefinitely.
                if err := job.UpdateTaskStatus(task, originalStatus); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Task &lt;%s/%s&gt; will be resynchronized after failing to revert status "+
                                "from %s to %s after failing to update Task on Node &lt;%s&gt;: %v",
                                task.Namespace, task.Name, task.Status, originalStatus, node.Name, err)
                        sc.resyncTask(task)
                }</span>
                <span class="cov0" title="0">return err</span>
        }

        <span class="cov0" title="0">p := task.Pod

        go func() </span><span class="cov0" title="0">{
                err := sc.Evictor.Evict(p, reason)
                if err != nil </span><span class="cov0" title="0">{
                        sc.resyncTask(task)
                }</span>
        }()

        <span class="cov0" title="0">podgroup := &amp;vcv1beta1.PodGroup{}
        if err := schedulingscheme.Scheme.Convert(&amp;job.PodGroup.PodGroup, podgroup, nil); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err)
                return err
        }</span>
        <span class="cov0" title="0">sc.Recorder.Eventf(podgroup, v1.EventTypeNormal, "Evict", reason)
        return nil</span>
}

// Bind binds task to the target host.
func (sc *SchedulerCache) Bind(taskInfo *schedulingapi.TaskInfo, hostname string) error <span class="cov8" title="1">{
        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        job, task, err := sc.findJobAndTask(taskInfo)

        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">node, found := sc.Nodes[hostname]
        if !found </span><span class="cov0" title="0">{
                return fmt.Errorf("failed to bind Task %v to host %v, host does not exist",
                        task.UID, hostname)
        }</span>

        <span class="cov8" title="1">originalStatus := task.Status
        if err := job.UpdateTaskStatus(task, schedulingapi.Binding); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        // Add task to the node.
        <span class="cov8" title="1">if err := node.AddTask(task); err != nil </span><span class="cov8" title="1">{
                // After failing to update task to a node we need to revert task status from Releasing,
                // otherwise task might be stuck in the Releasing state indefinitely.
                if err := job.UpdateTaskStatus(task, originalStatus); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Task &lt;%s/%s&gt; will be resynchronized after failing to revert status "+
                                "from %s to %s after failing to update Task on Node &lt;%s&gt;: %v",
                                task.Namespace, task.Name, task.Status, originalStatus, node.Name, err)
                        sc.resyncTask(task)
                }</span>
                <span class="cov8" title="1">return err</span>
        }

        <span class="cov8" title="1">p := task.Pod
        if !(task.TopologyPolicy == "" || task.TopologyPolicy == "none") </span><span class="cov0" title="0">{
                if err := sc.Binder.Bind(p, hostname); err != nil </span><span class="cov0" title="0">{
                        sc.resyncTask(task)
                }</span> else<span class="cov0" title="0"> {
                        sc.Recorder.Eventf(p, v1.EventTypeNormal, "Scheduled", "Successfully assigned %v/%v to %v", p.Namespace, p.Name, hostname)
                }</span>
        } else<span class="cov8" title="1"> {
                go func() </span><span class="cov8" title="1">{
                        if err := sc.Binder.Bind(p, hostname); err != nil </span><span class="cov0" title="0">{
                                sc.resyncTask(task)
                        }</span> else<span class="cov0" title="0"> {
                                sc.Recorder.Eventf(p, v1.EventTypeNormal, "Scheduled", "Successfully assigned %v/%v to %v", p.Namespace, p.Name, hostname)
                        }</span>
                }()
        }

        <span class="cov8" title="1">return nil</span>
}

// BindPodGroup binds job to silo cluster
func (sc *SchedulerCache) BindPodGroup(job *schedulingapi.JobInfo, cluster string) error <span class="cov0" title="0">{
        if _, err := sc.PodGroupBinder.Bind(job, cluster); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Bind job &lt;%s&gt; to cluster &lt;%s&gt; failed: %v", job.Name, cluster, err)
                return err
        }</span>
        <span class="cov0" title="0">return nil</span>
}

// GetPodVolumes get pod volume on the host
func (sc *SchedulerCache) GetPodVolumes(task *schedulingapi.TaskInfo, node *v1.Node) (*volumescheduling.PodVolumes, error) <span class="cov0" title="0">{
        return sc.VolumeBinder.GetPodVolumes(task, node)
}</span>

// AllocateVolumes allocates volume on the host to the task
func (sc *SchedulerCache) AllocateVolumes(task *schedulingapi.TaskInfo, hostname string, podVolumes *volumescheduling.PodVolumes) error <span class="cov0" title="0">{
        return sc.VolumeBinder.AllocateVolumes(task, hostname, podVolumes)
}</span>

// BindVolumes binds volumes to the task
func (sc *SchedulerCache) BindVolumes(task *schedulingapi.TaskInfo, podVolumes *volumescheduling.PodVolumes) error <span class="cov0" title="0">{
        return sc.VolumeBinder.BindVolumes(task, podVolumes)
}</span>

// Client returns the kubernetes clientSet
func (sc *SchedulerCache) Client() kubernetes.Interface <span class="cov0" title="0">{
        return sc.kubeClient
}</span>

// SharedInformerFactory returns the scheduler SharedInformerFactory
func (sc *SchedulerCache) SharedInformerFactory() informers.SharedInformerFactory <span class="cov0" title="0">{
        return sc.informerFactory
}</span>

// UpdateSchedulerNumaInfo used to update scheduler node cache NumaSchedulerInfo
func (sc *SchedulerCache) UpdateSchedulerNumaInfo(AllocatedSets map[string]schedulingapi.ResNumaSets) error <span class="cov0" title="0">{
        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        for nodeName, sets := range AllocatedSets </span><span class="cov0" title="0">{
                if _, found := sc.Nodes[nodeName]; !found </span><span class="cov0" title="0">{
                        continue</span>
                }

                <span class="cov0" title="0">numaInfo := sc.Nodes[nodeName].NumaSchedulerInfo
                if numaInfo == nil </span><span class="cov0" title="0">{
                        continue</span>
                }

                <span class="cov0" title="0">numaInfo.Allocate(sets)</span>
        }
        <span class="cov0" title="0">return nil</span>
}

// taskUnschedulable updates pod status of pending task
func (sc *SchedulerCache) taskUnschedulable(task *schedulingapi.TaskInfo, reason, message string) error <span class="cov0" title="0">{
        pod := task.Pod

        condition := &amp;v1.PodCondition{
                Type:    v1.PodScheduled,
                Status:  v1.ConditionFalse,
                Reason:  reason, // Add more reasons in order to distinguish more specific scenario of pending tasks
                Message: message,
        }

        if podConditionHaveUpdate(&amp;pod.Status, condition) </span><span class="cov0" title="0">{
                pod = pod.DeepCopy()

                // The reason field in 'Events' should be "FailedScheduling", there is not constants defined for this in
                // k8s core, so using the same string here.
                // The reason field in PodCondition can be "Unschedulable"
                sc.Recorder.Eventf(pod, v1.EventTypeWarning, "FailedScheduling", message)
                if _, err := sc.StatusUpdater.UpdatePodCondition(pod, condition); err != nil </span><span class="cov0" title="0">{
                        return err
                }</span>
        } else<span class="cov0" title="0"> {
                klog.V(4).Infof("task unscheduleable %s/%s, message: %s, skip by no condition update", pod.Namespace, pod.Name, message)
        }</span>

        <span class="cov0" title="0">return nil</span>
}

func (sc *SchedulerCache) deleteJob(job *schedulingapi.JobInfo) <span class="cov8" title="1">{
        klog.V(3).Infof("Try to delete Job &lt;%v:%v/%v&gt;", job.UID, job.Namespace, job.Name)

        sc.deletedJobs.AddRateLimited(job)
}</span>

func (sc *SchedulerCache) processCleanupJob() <span class="cov0" title="0">{
        obj, shutdown := sc.deletedJobs.Get()
        if shutdown </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov0" title="0">defer sc.deletedJobs.Done(obj)

        job, found := obj.(*schedulingapi.JobInfo)
        if !found </span><span class="cov0" title="0">{
                klog.Errorf("Failed to convert &lt;%v&gt; to *JobInfo", obj)
                return
        }</span>

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        if schedulingapi.JobTerminated(job) </span><span class="cov0" title="0">{
                delete(sc.Jobs, job.UID)
                klog.V(3).Infof("Job &lt;%v:%v/%v&gt; was deleted.", job.UID, job.Namespace, job.Name)
        }</span> else<span class="cov0" title="0"> {
                // Retry
                sc.deleteJob(job)
        }</span>
}

func (sc *SchedulerCache) resyncTask(task *schedulingapi.TaskInfo) <span class="cov0" title="0">{
        sc.errTasks.AddRateLimited(task)
}</span>

func (sc *SchedulerCache) processResyncTask() <span class="cov0" title="0">{
        obj, shutdown := sc.errTasks.Get()
        if shutdown </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov0" title="0">defer sc.errTasks.Done(obj)

        task, ok := obj.(*schedulingapi.TaskInfo)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("failed to convert %v to *schedulingapi.TaskInfo", obj)
                return
        }</span>

        <span class="cov0" title="0">if err := sc.syncTask(task); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to sync pod &lt;%v/%v&gt;, retry it.", task.Namespace, task.Name)
                sc.resyncTask(task)
        }</span>
}

// Snapshot returns the complete snapshot of the cluster from cache
func (sc *SchedulerCache) Snapshot() *schedulingapi.ClusterInfo <span class="cov0" title="0">{
        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        snapshot := &amp;schedulingapi.ClusterInfo{
                Nodes:          make(map[string]*schedulingapi.NodeInfo),
                Jobs:           make(map[schedulingapi.JobID]*schedulingapi.JobInfo),
                Queues:         make(map[schedulingapi.QueueID]*schedulingapi.QueueInfo),
                NamespaceInfo:  make(map[schedulingapi.NamespaceName]*schedulingapi.NamespaceInfo),
                RevocableNodes: make(map[string]*schedulingapi.NodeInfo),
                NodeList:       make([]string, len(sc.NodeList)),
        }

        copy(snapshot.NodeList, sc.NodeList)
        for _, value := range sc.Nodes </span><span class="cov0" title="0">{
                value.RefreshNumaSchedulerInfoByCrd()
        }</span>

        <span class="cov0" title="0">for _, value := range sc.Nodes </span><span class="cov0" title="0">{
                if !value.Ready() </span><span class="cov0" title="0">{
                        continue</span>
                }

                <span class="cov0" title="0">snapshot.Nodes[value.Name] = value.Clone()

                if value.RevocableZone != "" </span><span class="cov0" title="0">{
                        snapshot.RevocableNodes[value.Name] = snapshot.Nodes[value.Name]
                }</span>
        }

        <span class="cov0" title="0">for _, value := range sc.Queues </span><span class="cov0" title="0">{
                snapshot.Queues[value.UID] = value.Clone()
        }</span>

        <span class="cov0" title="0">var cloneJobLock sync.Mutex
        var wg sync.WaitGroup

        cloneJob := func(value *schedulingapi.JobInfo) </span><span class="cov0" title="0">{
                defer wg.Done()
                if value.PodGroup != nil </span><span class="cov0" title="0">{
                        value.Priority = sc.defaultPriority

                        priName := value.PodGroup.Spec.PriorityClassName
                        if priorityClass, found := sc.PriorityClasses[priName]; found </span><span class="cov0" title="0">{
                                value.Priority = priorityClass.Value
                        }</span>

                        <span class="cov0" title="0">klog.V(4).Infof("The priority of job &lt;%s/%s&gt; is &lt;%s/%d&gt;",
                                value.Namespace, value.Name, priName, value.Priority)</span>
                }

                <span class="cov0" title="0">clonedJob := value.Clone()

                cloneJobLock.Lock()
                snapshot.Jobs[value.UID] = clonedJob
                cloneJobLock.Unlock()</span>
        }

        <span class="cov0" title="0">for _, value := range sc.NamespaceCollection </span><span class="cov0" title="0">{
                info := value.Snapshot()
                snapshot.NamespaceInfo[info.Name] = info
                klog.V(4).Infof("Namespace %s has weight %v",
                        value.Name, info.GetWeight())
        }</span>

        <span class="cov0" title="0">for _, value := range sc.Jobs </span><span class="cov0" title="0">{
                // If no scheduling spec, does not handle it.
                if value.PodGroup == nil </span><span class="cov0" title="0">{
                        klog.V(4).Infof("The scheduling spec of Job &lt;%v:%s/%s&gt; is nil, ignore it.",
                                value.UID, value.Namespace, value.Name)

                        continue</span>
                }

                <span class="cov0" title="0">if _, found := snapshot.Queues[value.Queue]; !found </span><span class="cov0" title="0">{
                        klog.V(3).Infof("The Queue &lt;%v&gt; of Job &lt;%v/%v&gt; does not exist, ignore it.",
                                value.Queue, value.Namespace, value.Name)
                        continue</span>
                }

                <span class="cov0" title="0">wg.Add(1)
                go cloneJob(value)</span>
        }
        <span class="cov0" title="0">wg.Wait()

        klog.V(3).Infof("There are &lt;%d&gt; Jobs, &lt;%d&gt; Queues and &lt;%d&gt; Nodes in total for scheduling.",
                len(snapshot.Jobs), len(snapshot.Queues), len(snapshot.Nodes))

        return snapshot</span>
}

// String returns information about the cache in a string format
func (sc *SchedulerCache) String() string <span class="cov0" title="0">{
        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        str := "Cache:\n"

        if len(sc.Nodes) != 0 </span><span class="cov0" title="0">{
                str += "Nodes:\n"
                for _, n := range sc.Nodes </span><span class="cov0" title="0">{
                        str += fmt.Sprintf("\t %s: idle(%v) used(%v) allocatable(%v) pods(%d)\n",
                                n.Name, n.Idle, n.Used, n.Allocatable, len(n.Tasks))

                        i := 0
                        for _, p := range n.Tasks </span><span class="cov0" title="0">{
                                str += fmt.Sprintf("\t\t %d: %v\n", i, p)
                                i++
                        }</span>
                }
        }

        <span class="cov0" title="0">if len(sc.Jobs) != 0 </span><span class="cov0" title="0">{
                str += "Jobs:\n"
                for _, job := range sc.Jobs </span><span class="cov0" title="0">{
                        str += fmt.Sprintf("\t %s\n", job)
                }</span>
        }

        <span class="cov0" title="0">if len(sc.NamespaceCollection) != 0 </span><span class="cov0" title="0">{
                str += "Namespaces:\n"
                for _, ns := range sc.NamespaceCollection </span><span class="cov0" title="0">{
                        info := ns.Snapshot()
                        str += fmt.Sprintf("\t Namespace(%s) Weight(%v)\n",
                                info.Name, info.Weight)
                }</span>
        }

        <span class="cov0" title="0">if len(sc.NodeList) != 0 </span><span class="cov0" title="0">{
                str += fmt.Sprintf("NodeList: %v\n", sc.NodeList)
        }</span>

        <span class="cov0" title="0">return str</span>
}

// RecordJobStatusEvent records related events according to job status.
func (sc *SchedulerCache) RecordJobStatusEvent(job *schedulingapi.JobInfo) <span class="cov0" title="0">{
        pgUnschedulable := job.PodGroup != nil &amp;&amp;
                (job.PodGroup.Status.Phase == scheduling.PodGroupUnknown ||
                        job.PodGroup.Status.Phase == scheduling.PodGroupPending ||
                        job.PodGroup.Status.Phase == scheduling.PodGroupInqueue)

        // If pending or unschedulable, record unschedulable event.
        if pgUnschedulable </span><span class="cov0" title="0">{
                msg := fmt.Sprintf("%v/%v tasks in gang unschedulable: %v",
                        len(job.TaskStatusIndex[schedulingapi.Pending]),
                        len(job.Tasks),
                        job.FitError())
                sc.recordPodGroupEvent(job.PodGroup, v1.EventTypeWarning, string(scheduling.PodGroupUnschedulableType), msg)
        }</span> else<span class="cov0" title="0"> {
                sc.recordPodGroupEvent(job.PodGroup, v1.EventTypeNormal, string(scheduling.PodGroupScheduled), string(scheduling.PodGroupReady))
        }</span>

        <span class="cov0" title="0">baseErrorMessage := job.JobFitErrors
        if baseErrorMessage == "" </span><span class="cov0" title="0">{
                baseErrorMessage = schedulingapi.AllNodeUnavailableMsg
        }</span>
        // Update podCondition for tasks Allocated and Pending before job discarded
        <span class="cov0" title="0">for _, status := range []schedulingapi.TaskStatus{schedulingapi.Allocated, schedulingapi.Pending, schedulingapi.Pipelined} </span><span class="cov0" title="0">{
                for _, taskInfo := range job.TaskStatusIndex[status] </span><span class="cov0" title="0">{
                        reason, msg := job.TaskSchedulingReason(taskInfo.UID)
                        if len(msg) == 0 </span><span class="cov0" title="0">{
                                msg = baseErrorMessage
                        }</span>
                        <span class="cov0" title="0">if err := sc.taskUnschedulable(taskInfo, reason, msg); err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("Failed to update unschedulable task status &lt;%s/%s&gt;: %v",
                                        taskInfo.Namespace, taskInfo.Name, err)
                        }</span>
                }
        }
}

// UpdateJobStatus update the status of job and its tasks.
func (sc *SchedulerCache) UpdateJobStatus(job *schedulingapi.JobInfo, updatePG bool) (*schedulingapi.JobInfo, error) <span class="cov0" title="0">{
        if updatePG </span><span class="cov0" title="0">{
                pg, err := sc.StatusUpdater.UpdatePodGroup(job.PodGroup)
                if err != nil </span><span class="cov0" title="0">{
                        return nil, err
                }</span>
                <span class="cov0" title="0">job.PodGroup = pg</span>
        }

        <span class="cov0" title="0">sc.RecordJobStatusEvent(job)

        return job, nil</span>
}

func (sc *SchedulerCache) recordPodGroupEvent(podGroup *schedulingapi.PodGroup, eventType, reason, msg string) <span class="cov0" title="0">{
        if podGroup == nil </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov0" title="0">pg := &amp;vcv1beta1.PodGroup{}
        if err := schedulingscheme.Scheme.Convert(&amp;podGroup.PodGroup, pg, nil); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err)
                return
        }</span>
        <span class="cov0" title="0">sc.Recorder.Eventf(pg, eventType, reason, msg)</span>
}
</pre>
		
		<pre class="file" id="file54" style="display: none">/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
        "context"
        "fmt"
        "strconv"

        v1 "k8s.io/api/core/v1"
        "k8s.io/api/scheduling/v1beta1"
        "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/client-go/tools/cache"
        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"

        nodeinfov1alpha1 "volcano.sh/apis/pkg/apis/nodeinfo/v1alpha1"
        "volcano.sh/apis/pkg/apis/scheduling"
        "volcano.sh/apis/pkg/apis/scheduling/scheme"
        schedulingv1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/apis/pkg/apis/utils"
        schedulingapi "volcano.sh/volcano/pkg/scheduler/api"
)

func isTerminated(status schedulingapi.TaskStatus) bool <span class="cov8" title="1">{
        return status == schedulingapi.Succeeded || status == schedulingapi.Failed
}</span>

// getOrCreateJob will return corresponding Job for pi if it exists, or it will create a Job and return it if
// pi.Pod.Spec.SchedulerName is same as volcano scheduler's name, otherwise it will return nil.
func (sc *SchedulerCache) getOrCreateJob(pi *schedulingapi.TaskInfo) *schedulingapi.JobInfo <span class="cov8" title="1">{
        if len(pi.Job) == 0 </span><span class="cov8" title="1">{
                if pi.Pod.Spec.SchedulerName != sc.schedulerName </span><span class="cov8" title="1">{
                        klog.V(4).Infof("Pod %s/%s will not not scheduled by %s, skip creating PodGroup and Job for it",
                                pi.Pod.Namespace, pi.Pod.Name, sc.schedulerName)
                }</span>
                <span class="cov8" title="1">return nil</span>
        }

        <span class="cov8" title="1">if _, found := sc.Jobs[pi.Job]; !found </span><span class="cov8" title="1">{
                sc.Jobs[pi.Job] = schedulingapi.NewJobInfo(pi.Job)
        }</span>

        <span class="cov8" title="1">return sc.Jobs[pi.Job]</span>
}

func (sc *SchedulerCache) addTask(pi *schedulingapi.TaskInfo) error <span class="cov8" title="1">{
        if len(pi.NodeName) != 0 </span><span class="cov8" title="1">{
                if _, found := sc.Nodes[pi.NodeName]; !found </span><span class="cov0" title="0">{
                        sc.Nodes[pi.NodeName] = schedulingapi.NewNodeInfo(nil)
                        sc.Nodes[pi.NodeName].Name = pi.NodeName
                }</span>

                <span class="cov8" title="1">node := sc.Nodes[pi.NodeName]
                if !isTerminated(pi.Status) </span><span class="cov8" title="1">{
                        if err := node.AddTask(pi); err != nil </span><span class="cov0" title="0">{
                                return err
                        }</span>
                } else<span class="cov8" title="1"> {
                        klog.V(4).Infof("Pod &lt;%v/%v&gt; is in status %s.", pi.Namespace, pi.Name, pi.Status.String())
                }</span>
        }

        <span class="cov8" title="1">job := sc.getOrCreateJob(pi)
        if job != nil </span><span class="cov8" title="1">{
                job.AddTaskInfo(pi)
        }</span>

        <span class="cov8" title="1">return nil</span>
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) addPod(pod *v1.Pod) error <span class="cov8" title="1">{
        pi := schedulingapi.NewTaskInfo(pod)

        return sc.addTask(pi)
}</span>

func (sc *SchedulerCache) syncTask(oldTask *schedulingapi.TaskInfo) error <span class="cov0" title="0">{
        newPod, err := sc.kubeClient.CoreV1().Pods(oldTask.Namespace).Get(context.TODO(), oldTask.Name, metav1.GetOptions{})
        if err != nil </span><span class="cov0" title="0">{
                if errors.IsNotFound(err) </span><span class="cov0" title="0">{
                        err := sc.deleteTask(oldTask)
                        if err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("Failed to delete Pod &lt;%v/%v&gt; and remove from cache: %s", oldTask.Namespace, oldTask.Name, err.Error())
                                return err
                        }</span>
                        <span class="cov0" title="0">klog.V(3).Infof("Pod &lt;%v/%v&gt; was deleted, removed from cache.", oldTask.Namespace, oldTask.Name)

                        return nil</span>
                }
                <span class="cov0" title="0">return fmt.Errorf("failed to get Pod &lt;%v/%v&gt;: err %v", oldTask.Namespace, oldTask.Name, err)</span>
        }

        <span class="cov0" title="0">newTask := schedulingapi.NewTaskInfo(newPod)

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()
        return sc.updateTask(oldTask, newTask)</span>
}

func (sc *SchedulerCache) updateTask(oldTask, newTask *schedulingapi.TaskInfo) error <span class="cov8" title="1">{
        if err := sc.deleteTask(oldTask); err != nil </span><span class="cov0" title="0">{
                klog.Warningf("Failed to delete task: %v", err)
        }</span>

        <span class="cov8" title="1">return sc.addTask(newTask)</span>
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) updatePod(oldPod, newPod *v1.Pod) error <span class="cov8" title="1">{
        if err := sc.deletePod(oldPod); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>
        //when delete pod, the ownerreference of pod will be set nil,just as orphan pod
        <span class="cov8" title="1">if len(utils.GetController(newPod)) == 0 </span><span class="cov0" title="0">{
                newPod.OwnerReferences = oldPod.OwnerReferences
        }</span>
        <span class="cov8" title="1">return sc.addPod(newPod)</span>
}

func (sc *SchedulerCache) deleteTask(pi *schedulingapi.TaskInfo) error <span class="cov8" title="1">{
        var jobErr, nodeErr, numaErr error

        if len(pi.Job) != 0 </span><span class="cov0" title="0">{
                if job, found := sc.Jobs[pi.Job]; found </span><span class="cov0" title="0">{
                        jobErr = job.DeleteTaskInfo(pi)
                }</span> else<span class="cov0" title="0"> {
                        jobErr = fmt.Errorf("failed to find Job &lt;%v&gt; for Task %v/%v",
                                pi.Job, pi.Namespace, pi.Name)
                }</span>
        }

        <span class="cov8" title="1">if len(pi.NodeName) != 0 </span><span class="cov8" title="1">{
                node := sc.Nodes[pi.NodeName]
                if node != nil </span><span class="cov8" title="1">{
                        nodeErr = node.RemoveTask(pi)
                }</span>
        }

        <span class="cov8" title="1">if jobErr != nil || nodeErr != nil </span><span class="cov0" title="0">{
                return schedulingapi.MergeErrors(jobErr, nodeErr, numaErr)
        }</span>

        <span class="cov8" title="1">return nil</span>
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) deletePod(pod *v1.Pod) error <span class="cov8" title="1">{
        pi := schedulingapi.NewTaskInfo(pod)

        // Delete the Task in cache to handle Binding status.
        task := pi
        if job, found := sc.Jobs[pi.Job]; found </span><span class="cov0" title="0">{
                if t, found := job.Tasks[pi.UID]; found </span><span class="cov0" title="0">{
                        task = t
                }</span>
        }
        <span class="cov8" title="1">if err := sc.deleteTask(task); err != nil </span><span class="cov0" title="0">{
                klog.Warningf("Failed to delete task: %v", err)
        }</span>

        // If job was terminated, delete it.
        <span class="cov8" title="1">if job, found := sc.Jobs[pi.Job]; found &amp;&amp; schedulingapi.JobTerminated(job) </span><span class="cov0" title="0">{
                sc.deleteJob(job)
        }</span>

        <span class="cov8" title="1">return nil</span>
}

// AddPod add pod to scheduler cache
func (sc *SchedulerCache) AddPod(obj interface{}) <span class="cov8" title="1">{
        pod, ok := obj.(*v1.Pod)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert to *v1.Pod: %v", obj)
                return
        }</span>

        <span class="cov8" title="1">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        err := sc.addPod(pod)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to add pod &lt;%s/%s&gt; into cache: %v",
                        pod.Namespace, pod.Name, err)
                return
        }</span>
        <span class="cov8" title="1">klog.V(3).Infof("Added pod &lt;%s/%v&gt; into cache.", pod.Namespace, pod.Name)</span>
}

// UpdatePod update pod to scheduler cache
func (sc *SchedulerCache) UpdatePod(oldObj, newObj interface{}) <span class="cov0" title="0">{
        oldPod, ok := oldObj.(*v1.Pod)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert oldObj to *v1.Pod: %v", oldObj)
                return
        }</span>
        <span class="cov0" title="0">newPod, ok := newObj.(*v1.Pod)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert newObj to *v1.Pod: %v", newObj)
                return
        }</span>

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        err := sc.updatePod(oldPod, newPod)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to update pod %v in cache: %v", oldPod.Name, err)
                return
        }</span>

        <span class="cov0" title="0">klog.V(4).Infof("Updated pod &lt;%s/%v&gt; in cache.", oldPod.Namespace, oldPod.Name)</span>
}

// DeletePod delete pod from scheduler cache
func (sc *SchedulerCache) DeletePod(obj interface{}) <span class="cov0" title="0">{
        var pod *v1.Pod
        switch t := obj.(type) </span>{
        case *v1.Pod:<span class="cov0" title="0">
                pod = t</span>
        case cache.DeletedFinalStateUnknown:<span class="cov0" title="0">
                var ok bool
                pod, ok = t.Obj.(*v1.Pod)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Cannot convert to *v1.Pod: %v", t.Obj)
                        return
                }</span>
        default:<span class="cov0" title="0">
                klog.Errorf("Cannot convert to *v1.Pod: %v", t)
                return</span>
        }

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        err := sc.deletePod(pod)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to delete pod %v from cache: %v", pod.Name, err)
                return
        }</span>

        <span class="cov0" title="0">klog.V(3).Infof("Deleted pod &lt;%s/%v&gt; from cache.", pod.Namespace, pod.Name)</span>
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) addNode(node *v1.Node) error <span class="cov8" title="1">{
        if sc.Nodes[node.Name] != nil </span><span class="cov0" title="0">{
                sc.Nodes[node.Name].SetNode(node)
        }</span> else<span class="cov8" title="1"> {
                sc.Nodes[node.Name] = schedulingapi.NewNodeInfo(node)
        }</span>
        <span class="cov8" title="1">return nil</span>
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) updateNode(oldNode, newNode *v1.Node) error <span class="cov0" title="0">{
        if sc.Nodes[newNode.Name] != nil </span><span class="cov0" title="0">{
                sc.Nodes[newNode.Name].SetNode(newNode)
                return nil
        }</span>

        <span class="cov0" title="0">return fmt.Errorf("node &lt;%s&gt; does not exist", newNode.Name)</span>
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) deleteNode(node *v1.Node) error <span class="cov8" title="1">{
        if _, ok := sc.Nodes[node.Name]; !ok </span><span class="cov0" title="0">{
                return fmt.Errorf("node &lt;%s&gt; does not exist", node.Name)
        }</span>

        <span class="cov8" title="1">numaInfo := sc.Nodes[node.Name].NumaInfo
        if numaInfo != nil </span><span class="cov0" title="0">{
                klog.V(3).Infof("delete numatopo &lt;%s/%s&gt;", numaInfo.Namespace, numaInfo.Name)
                err := sc.vcClient.NodeinfoV1alpha1().Numatopologies().Delete(context.TODO(), numaInfo.Name, metav1.DeleteOptions{})
                if err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("delete numatopo &lt;%s/%s&gt; failed.", numaInfo.Namespace, numaInfo.Name)
                }</span>
        }

        <span class="cov8" title="1">delete(sc.Nodes, node.Name)

        return nil</span>
}

// AddNode add node to scheduler cache
func (sc *SchedulerCache) AddNode(obj interface{}) <span class="cov8" title="1">{
        node, ok := obj.(*v1.Node)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert to *v1.Node: %v", obj)
                return
        }</span>

        <span class="cov8" title="1">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        err := sc.addNode(node)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to add node %s into cache: %v", node.Name, err)
                return
        }</span>
        <span class="cov8" title="1">sc.NodeList = append(sc.NodeList, node.Name)</span>
}

// UpdateNode update node to scheduler cache
func (sc *SchedulerCache) UpdateNode(oldObj, newObj interface{}) <span class="cov0" title="0">{
        oldNode, ok := oldObj.(*v1.Node)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert oldObj to *v1.Node: %v", oldObj)
                return
        }</span>
        <span class="cov0" title="0">newNode, ok := newObj.(*v1.Node)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert newObj to *v1.Node: %v", newObj)
                return
        }</span>

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        err := sc.updateNode(oldNode, newNode)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to update node %v in cache: %v", oldNode.Name, err)
                return
        }</span>
}

// DeleteNode delete node from scheduler cache
func (sc *SchedulerCache) DeleteNode(obj interface{}) <span class="cov8" title="1">{
        var node *v1.Node
        switch t := obj.(type) </span>{
        case *v1.Node:<span class="cov8" title="1">
                node = t</span>
        case cache.DeletedFinalStateUnknown:<span class="cov0" title="0">
                var ok bool
                node, ok = t.Obj.(*v1.Node)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Cannot convert to *v1.Node: %v", t.Obj)
                        return
                }</span>
        default:<span class="cov0" title="0">
                klog.Errorf("Cannot convert to *v1.Node: %v", t)
                return</span>
        }

        <span class="cov8" title="1">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        err := sc.deleteNode(node)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to delete node %s from cache: %v", node.Name, err)
                return
        }</span>

        <span class="cov8" title="1">for i, name := range sc.NodeList </span><span class="cov8" title="1">{
                if name == node.Name </span><span class="cov8" title="1">{
                        sc.NodeList = append(sc.NodeList[:i], sc.NodeList[i+1:]...)
                        break</span>
                }
        }
}

func getJobID(pg *schedulingapi.PodGroup) schedulingapi.JobID <span class="cov8" title="1">{
        return schedulingapi.JobID(fmt.Sprintf("%s/%s", pg.Namespace, pg.Name))
}</span>

// Assumes that lock is already acquired.
func (sc *SchedulerCache) setPodGroup(ss *schedulingapi.PodGroup) error <span class="cov8" title="1">{
        job := getJobID(ss)
        if _, found := sc.Jobs[job]; !found </span><span class="cov8" title="1">{
                sc.Jobs[job] = schedulingapi.NewJobInfo(job)
        }</span>

        <span class="cov8" title="1">sc.Jobs[job].SetPodGroup(ss)

        // TODO(k82cn): set default queue in admission.
        if len(ss.Spec.Queue) == 0 </span><span class="cov8" title="1">{
                sc.Jobs[job].Queue = schedulingapi.QueueID(sc.defaultQueue)
        }</span>

        <span class="cov8" title="1">return nil</span>
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) updatePodGroup(newPodGroup *schedulingapi.PodGroup) error <span class="cov0" title="0">{
        return sc.setPodGroup(newPodGroup)
}</span>

// Assumes that lock is already acquired.
func (sc *SchedulerCache) deletePodGroup(id schedulingapi.JobID) error <span class="cov8" title="1">{
        job, found := sc.Jobs[id]
        if !found </span><span class="cov0" title="0">{
                return fmt.Errorf("can not found job %v", id)
        }</span>

        // Unset SchedulingSpec
        <span class="cov8" title="1">job.UnsetPodGroup()

        sc.deleteJob(job)

        return nil</span>
}

// AddPodGroupV1beta1 add podgroup to scheduler cache
func (sc *SchedulerCache) AddPodGroupV1beta1(obj interface{}) <span class="cov8" title="1">{
        ss, ok := obj.(*schedulingv1.PodGroup)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert to *schedulingv1.PodGroup: %v", obj)
                return
        }</span>

        <span class="cov8" title="1">podgroup := scheduling.PodGroup{}
        if err := scheme.Scheme.Convert(ss, &amp;podgroup, nil); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to convert podgroup from %T to %T", ss, podgroup)
                return
        }</span>

        <span class="cov8" title="1">pg := &amp;schedulingapi.PodGroup{PodGroup: podgroup, Version: schedulingapi.PodGroupVersionV1Beta1}
        klog.V(4).Infof("Add PodGroup(%s) into cache, spec(%#v)", ss.Name, ss.Spec)

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        if err := sc.setPodGroup(pg); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to add PodGroup %s into cache: %v", ss.Name, err)
                return
        }</span>
}

// UpdatePodGroupV1beta1 add podgroup to scheduler cache
func (sc *SchedulerCache) UpdatePodGroupV1beta1(oldObj, newObj interface{}) <span class="cov8" title="1">{
        oldSS, ok := oldObj.(*schedulingv1.PodGroup)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert oldObj to *schedulingv1.SchedulingSpec: %v", oldObj)
                return
        }</span>
        <span class="cov8" title="1">newSS, ok := newObj.(*schedulingv1.PodGroup)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert newObj to *schedulingv1.SchedulingSpec: %v", newObj)
                return
        }</span>

        <span class="cov8" title="1">if oldSS.ResourceVersion == newSS.ResourceVersion </span><span class="cov8" title="1">{
                return
        }</span>

        <span class="cov0" title="0">podgroup := scheduling.PodGroup{}
        if err := scheme.Scheme.Convert(newSS, &amp;podgroup, nil); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to convert podgroup from %T to %T", newSS, podgroup)
                return
        }</span>

        <span class="cov0" title="0">pg := &amp;schedulingapi.PodGroup{PodGroup: podgroup, Version: schedulingapi.PodGroupVersionV1Beta1}

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        if err := sc.updatePodGroup(pg); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to update SchedulingSpec %s into cache: %v", pg.Name, err)
                return
        }</span>
}

// DeletePodGroupV1beta1 delete podgroup from scheduler cache
func (sc *SchedulerCache) DeletePodGroupV1beta1(obj interface{}) <span class="cov8" title="1">{
        var ss *schedulingv1.PodGroup
        switch t := obj.(type) </span>{
        case *schedulingv1.PodGroup:<span class="cov8" title="1">
                ss = t</span>
        case cache.DeletedFinalStateUnknown:<span class="cov0" title="0">
                var ok bool
                ss, ok = t.Obj.(*schedulingv1.PodGroup)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Cannot convert to podgroup: %v", t.Obj)
                        return
                }</span>
        default:<span class="cov0" title="0">
                klog.Errorf("Cannot convert to podgroup: %v", t)
                return</span>
        }

        <span class="cov8" title="1">jobID := schedulingapi.JobID(fmt.Sprintf("%s/%s", ss.Namespace, ss.Name))

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        if err := sc.deletePodGroup(jobID); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to delete podgroup %s from cache: %v", ss.Name, err)
                return
        }</span>
}

// AddQueueV1beta1 add queue to scheduler cache
func (sc *SchedulerCache) AddQueueV1beta1(obj interface{}) <span class="cov8" title="1">{
        ss, ok := obj.(*schedulingv1.Queue)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", obj)
                return
        }</span>

        <span class="cov8" title="1">queue := &amp;scheduling.Queue{}
        if err := scheme.Scheme.Convert(ss, queue, nil); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to convert queue from %T to %T", ss, queue)
                return
        }</span>

        <span class="cov8" title="1">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        klog.V(4).Infof("Add Queue(%s) into cache, spec(%#v)", ss.Name, ss.Spec)
        sc.addQueue(queue)</span>
}

// UpdateQueueV1beta1 update queue to scheduler cache
func (sc *SchedulerCache) UpdateQueueV1beta1(oldObj, newObj interface{}) <span class="cov8" title="1">{
        oldSS, ok := oldObj.(*schedulingv1.Queue)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert oldObj to *schedulingv1.Queue: %v", oldObj)
                return
        }</span>
        <span class="cov8" title="1">newSS, ok := newObj.(*schedulingv1.Queue)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert newObj to *schedulingv1.Queue: %v", newObj)
                return
        }</span>

        <span class="cov8" title="1">if oldSS.ResourceVersion == newSS.ResourceVersion </span><span class="cov8" title="1">{
                return
        }</span>

        <span class="cov0" title="0">newQueue := &amp;scheduling.Queue{}
        if err := scheme.Scheme.Convert(newSS, newQueue, nil); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to convert queue from %T to %T", newSS, newQueue)
                return
        }</span>

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()
        sc.updateQueue(newQueue)</span>
}

// DeleteQueueV1beta1 delete queue from the scheduler cache
func (sc *SchedulerCache) DeleteQueueV1beta1(obj interface{}) <span class="cov8" title="1">{
        var ss *schedulingv1.Queue
        switch t := obj.(type) </span>{
        case *schedulingv1.Queue:<span class="cov8" title="1">
                ss = t</span>
        case cache.DeletedFinalStateUnknown:<span class="cov0" title="0">
                var ok bool
                ss, ok = t.Obj.(*schedulingv1.Queue)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t.Obj)
                        return
                }</span>
        default:<span class="cov0" title="0">
                klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t)
                return</span>
        }

        <span class="cov8" title="1">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()
        sc.deleteQueue(schedulingapi.QueueID(ss.Name))</span>
}

func (sc *SchedulerCache) addQueue(queue *scheduling.Queue) <span class="cov8" title="1">{
        qi := schedulingapi.NewQueueInfo(queue)
        sc.Queues[qi.UID] = qi
}</span>

func (sc *SchedulerCache) updateQueue(queue *scheduling.Queue) <span class="cov0" title="0">{
        sc.addQueue(queue)
}</span>

func (sc *SchedulerCache) deleteQueue(id schedulingapi.QueueID) <span class="cov8" title="1">{
        delete(sc.Queues, id)
}</span>

//DeletePriorityClass delete priorityclass from the scheduler cache
func (sc *SchedulerCache) DeletePriorityClass(obj interface{}) <span class="cov0" title="0">{
        var ss *v1beta1.PriorityClass
        switch t := obj.(type) </span>{
        case *v1beta1.PriorityClass:<span class="cov0" title="0">
                ss = t</span>
        case cache.DeletedFinalStateUnknown:<span class="cov0" title="0">
                var ok bool
                ss, ok = t.Obj.(*v1beta1.PriorityClass)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj)
                        return
                }</span>
        default:<span class="cov0" title="0">
                klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t)
                return</span>
        }

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        sc.deletePriorityClass(ss)</span>
}

//UpdatePriorityClass update priorityclass to scheduler cache
func (sc *SchedulerCache) UpdatePriorityClass(oldObj, newObj interface{}) <span class="cov0" title="0">{
        oldSS, ok := oldObj.(*v1beta1.PriorityClass)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert oldObj to *v1beta1.PriorityClass: %v", oldObj)

                return
        }</span>

        <span class="cov0" title="0">newSS, ok := newObj.(*v1beta1.PriorityClass)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert newObj to *v1beta1.PriorityClass: %v", newObj)
                return
        }</span>

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        sc.deletePriorityClass(oldSS)
        sc.addPriorityClass(newSS)</span>
}

//AddPriorityClass add priorityclass to scheduler cache
func (sc *SchedulerCache) AddPriorityClass(obj interface{}) <span class="cov0" title="0">{
        var ss *v1beta1.PriorityClass
        switch t := obj.(type) </span>{
        case *v1beta1.PriorityClass:<span class="cov0" title="0">
                ss = t</span>
        case cache.DeletedFinalStateUnknown:<span class="cov0" title="0">
                var ok bool
                ss, ok = t.Obj.(*v1beta1.PriorityClass)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj)
                        return
                }</span>
        default:<span class="cov0" title="0">
                klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t)
                return</span>
        }

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        sc.addPriorityClass(ss)</span>
}

func (sc *SchedulerCache) deletePriorityClass(pc *v1beta1.PriorityClass) <span class="cov0" title="0">{
        if pc.GlobalDefault </span><span class="cov0" title="0">{
                sc.defaultPriorityClass = nil
                sc.defaultPriority = 0
        }</span>

        <span class="cov0" title="0">delete(sc.PriorityClasses, pc.Name)</span>
}

func (sc *SchedulerCache) addPriorityClass(pc *v1beta1.PriorityClass) <span class="cov0" title="0">{
        if pc.GlobalDefault </span><span class="cov0" title="0">{
                if sc.defaultPriorityClass != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Updated default priority class from &lt;%s&gt; to &lt;%s&gt; forcefully.",
                                sc.defaultPriorityClass.Name, pc.Name)
                }</span>
                <span class="cov0" title="0">sc.defaultPriorityClass = pc
                sc.defaultPriority = pc.Value</span>
        }

        <span class="cov0" title="0">sc.PriorityClasses[pc.Name] = pc</span>
}

func (sc *SchedulerCache) updateResourceQuota(quota *v1.ResourceQuota) <span class="cov0" title="0">{
        collection, ok := sc.NamespaceCollection[quota.Namespace]
        if !ok </span><span class="cov0" title="0">{
                collection = schedulingapi.NewNamespaceCollection(quota.Namespace)
                sc.NamespaceCollection[quota.Namespace] = collection
        }</span>

        <span class="cov0" title="0">collection.Update(quota)</span>
}

func (sc *SchedulerCache) deleteResourceQuota(quota *v1.ResourceQuota) <span class="cov0" title="0">{
        collection, ok := sc.NamespaceCollection[quota.Namespace]
        if !ok </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov0" title="0">collection.Delete(quota)</span>
}

// DeleteResourceQuota delete ResourceQuota from the scheduler cache
func (sc *SchedulerCache) DeleteResourceQuota(obj interface{}) <span class="cov0" title="0">{
        var r *v1.ResourceQuota
        switch t := obj.(type) </span>{
        case *v1.ResourceQuota:<span class="cov0" title="0">
                r = t</span>
        case cache.DeletedFinalStateUnknown:<span class="cov0" title="0">
                var ok bool
                r, ok = t.Obj.(*v1.ResourceQuota)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t.Obj)
                        return
                }</span>
        default:<span class="cov0" title="0">
                klog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t)
                return</span>
        }

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        klog.V(3).Infof("Delete ResourceQuota &lt;%s/%v&gt; in cache", r.Namespace, r.Name)
        sc.deleteResourceQuota(r)</span>
}

// UpdateResourceQuota update ResourceQuota to scheduler cache
func (sc *SchedulerCache) UpdateResourceQuota(oldObj, newObj interface{}) <span class="cov0" title="0">{
        newR, ok := newObj.(*v1.ResourceQuota)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert newObj to *v1.ResourceQuota: %v", newObj)
                return
        }</span>

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        klog.V(3).Infof("Update ResourceQuota &lt;%s/%v&gt; in cache, with spec: %v.", newR.Namespace, newR.Name, newR.Spec.Hard)
        sc.updateResourceQuota(newR)</span>
}

// AddResourceQuota add ResourceQuota to scheduler cache
func (sc *SchedulerCache) AddResourceQuota(obj interface{}) <span class="cov0" title="0">{
        var r *v1.ResourceQuota
        switch t := obj.(type) </span>{
        case *v1.ResourceQuota:<span class="cov0" title="0">
                r = t</span>
        default:<span class="cov0" title="0">
                klog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t)
                return</span>
        }

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        klog.V(3).Infof("Add ResourceQuota &lt;%s/%v&gt; in cache, with spec: %v.", r.Namespace, r.Name, r.Spec.Hard)
        sc.updateResourceQuota(r)</span>
}

func getNumaInfo(srcInfo *nodeinfov1alpha1.Numatopology) *schedulingapi.NumatopoInfo <span class="cov0" title="0">{
        numaInfo := &amp;schedulingapi.NumatopoInfo{
                Namespace:   srcInfo.Namespace,
                Name:        srcInfo.Name,
                Policies:    make(map[nodeinfov1alpha1.PolicyName]string),
                NumaResMap:  make(map[string]*schedulingapi.ResourceInfo),
                CPUDetail:   topology.CPUDetails{},
                ResReserved: make(v1.ResourceList),
        }

        policies := srcInfo.Spec.Policies
        for name, policy := range policies </span><span class="cov0" title="0">{
                numaInfo.Policies[name] = policy
        }</span>

        <span class="cov0" title="0">numaResMap := srcInfo.Spec.NumaResMap
        for name, resInfo := range numaResMap </span><span class="cov0" title="0">{
                tmp := schedulingapi.ResourceInfo{}
                tmp.Capacity = resInfo.Capacity
                tmp.Allocatable = cpuset.MustParse(resInfo.Allocatable)
                numaInfo.NumaResMap[name] = &amp;tmp
        }</span>

        <span class="cov0" title="0">cpuDetail := srcInfo.Spec.CPUDetail
        for key, detail := range cpuDetail </span><span class="cov0" title="0">{
                cpuID, _ := strconv.Atoi(key)
                numaInfo.CPUDetail[cpuID] = topology.CPUInfo{
                        NUMANodeID: detail.NUMANodeID,
                        SocketID:   detail.SocketID,
                        CoreID:     detail.CoreID,
                }
        }</span>

        <span class="cov0" title="0">resReserved, err := schedulingapi.ParseResourceList(srcInfo.Spec.ResReserved)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("ParseResourceList failed, err=%v", err)
        }</span> else<span class="cov0" title="0"> {
                numaInfo.ResReserved = resReserved
        }</span>

        <span class="cov0" title="0">return numaInfo</span>
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) addNumaInfo(info *nodeinfov1alpha1.Numatopology) error <span class="cov0" title="0">{
        if sc.Nodes[info.Name] == nil </span><span class="cov0" title="0">{
                sc.Nodes[info.Name] = schedulingapi.NewNodeInfo(nil)
                sc.Nodes[info.Name].Name = info.Name
        }</span>

        <span class="cov0" title="0">if sc.Nodes[info.Name].NumaInfo == nil </span><span class="cov0" title="0">{
                sc.Nodes[info.Name].NumaInfo = getNumaInfo(info)
        }</span>

        <span class="cov0" title="0">newLocalInfo := getNumaInfo(info)
        if sc.Nodes[info.Name].NumaInfo.Compare(newLocalInfo) </span><span class="cov0" title="0">{
                sc.Nodes[info.Name].NumaChgFlag = schedulingapi.NumaInfoMoreFlag
        }</span> else<span class="cov0" title="0"> {
                sc.Nodes[info.Name].NumaChgFlag = schedulingapi.NumaInfoLessFlag
        }</span>

        <span class="cov0" title="0">sc.Nodes[info.Name].NumaInfo = newLocalInfo

        for resName, NumaResInfo := range sc.Nodes[info.Name].NumaInfo.NumaResMap </span><span class="cov0" title="0">{
                klog.V(3).Infof("resource %s Allocatable %v on node[%s] into cache", resName, NumaResInfo, info.Name)
        }</span>

        <span class="cov0" title="0">klog.V(3).Infof("Policies %v on node[%s] into cache, change= %v",
                sc.Nodes[info.Name].NumaInfo.Policies, info.Name, sc.Nodes[info.Name].NumaChgFlag)
        return nil</span>
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) deleteNumaInfo(info *nodeinfov1alpha1.Numatopology) <span class="cov0" title="0">{
        if sc.Nodes[info.Name] != nil </span><span class="cov0" title="0">{
                sc.Nodes[info.Name].NumaInfo = nil
                sc.Nodes[info.Name].NumaChgFlag = schedulingapi.NumaInfoResetFlag
                klog.V(3).Infof("delete numainfo in cahce for node&lt;%s&gt;", info.Name)
        }</span>
}

// AddNumaInfoV1alpha1 add numa information to scheduler cache
func (sc *SchedulerCache) AddNumaInfoV1alpha1(obj interface{}) <span class="cov0" title="0">{
        ss, ok := obj.(*nodeinfov1alpha1.Numatopology)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert oldObj to *nodeinfov1alpha1.Numatopology: %v", obj)
                return
        }</span>

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        sc.addNumaInfo(ss)</span>
}

// UpdateNumaInfoV1alpha1 update numa information to scheduler cache
func (sc *SchedulerCache) UpdateNumaInfoV1alpha1(oldObj, newObj interface{}) <span class="cov0" title="0">{
        ss, ok := newObj.(*nodeinfov1alpha1.Numatopology)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Cannot convert oldObj to *nodeinfov1alpha1.Numatopology: %v", newObj)
                return
        }</span>

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()
        sc.addNumaInfo(ss)
        klog.V(3).Infof("update numaInfo&lt;%s&gt; in cahce, with spec: Policy: %v, resMap: %v", ss.Name, ss.Spec.Policies, ss.Spec.NumaResMap)</span>
}

// DeleteNumaInfoV1alpha1 delete numa information from scheduler cache
func (sc *SchedulerCache) DeleteNumaInfoV1alpha1(obj interface{}) <span class="cov0" title="0">{
        var ss *nodeinfov1alpha1.Numatopology
        switch t := obj.(type) </span>{
        case *nodeinfov1alpha1.Numatopology:<span class="cov0" title="0">
                ss = t</span>
        case cache.DeletedFinalStateUnknown:<span class="cov0" title="0">
                var ok bool
                ss, ok = t.Obj.(*nodeinfov1alpha1.Numatopology)
                if !ok </span><span class="cov0" title="0">{
                        klog.Errorf("Cannot convert to Numatopo: %v", t.Obj)
                        return
                }</span>
        default:<span class="cov0" title="0">
                klog.Errorf("Cannot convert to Numatopo: %v", t)
                return</span>
        }

        <span class="cov0" title="0">sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        sc.deleteNumaInfo(ss)
        klog.V(3).Infof("Delete numaInfo&lt;%s&gt; from cahce, with spec: Policy: %v, resMap: %v", ss.Name, ss.Spec.Policies, ss.Spec.NumaResMap)</span>
}
</pre>
		
		<pre class="file" id="file55" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import v1 "k8s.io/api/core/v1"

// responsibleForPod returns true if the pod has asked to be scheduled by the given scheduler.
func responsibleForPod(pod *v1.Pod, schedulerName string) bool <span class="cov0" title="0">{
        return schedulerName == pod.Spec.SchedulerName
}</span>
</pre>
		
		<pre class="file" id="file56" style="display: none">/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
        "strconv"

        "volcano.sh/volcano/pkg/scheduler/conf"

        "k8s.io/klog"
)

// Arguments map
type Arguments map[string]string

// GetInt get the integer value from string
func (a Arguments) GetInt(ptr *int, key string) <span class="cov8" title="1">{
        if ptr == nil </span><span class="cov8" title="1">{
                return
        }</span>

        <span class="cov8" title="1">argv, ok := a[key]
        if !ok || argv == "" </span><span class="cov8" title="1">{
                return
        }</span>

        <span class="cov8" title="1">value, err := strconv.Atoi(argv)
        if err != nil </span><span class="cov8" title="1">{
                klog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err)
                return
        }</span>

        <span class="cov8" title="1">*ptr = value</span>
}

// GetFloat64 get the float64 value from string
func (a Arguments) GetFloat64(ptr *float64, key string) <span class="cov8" title="1">{
        if ptr == nil </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov8" title="1">argv, ok := a[key]
        if !ok || len(argv) == 0 </span><span class="cov8" title="1">{
                return
        }</span>

        <span class="cov8" title="1">value, err := strconv.ParseFloat(argv, 64)
        if err != nil </span><span class="cov8" title="1">{
                klog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err)
                return
        }</span>

        <span class="cov8" title="1">*ptr = value</span>
}

// GetBool get the bool value from string
func (a Arguments) GetBool(ptr *bool, key string) <span class="cov0" title="0">{
        if ptr == nil </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov0" title="0">argv, ok := a[key]
        if !ok || argv == "" </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov0" title="0">value, err := strconv.ParseBool(argv)
        if err != nil </span><span class="cov0" title="0">{
                klog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err)
                return
        }</span>

        <span class="cov0" title="0">*ptr = value</span>
}

// GetArgOfActionFromConf return argument of action reading from configuration of schedule
func GetArgOfActionFromConf(configurations []conf.Configuration, actionName string) Arguments <span class="cov8" title="1">{
        for _, c := range configurations </span><span class="cov8" title="1">{
                if c.Name == actionName </span><span class="cov8" title="1">{
                        return c.Arguments
                }</span>
        }

        <span class="cov8" title="1">return nil</span>
}
</pre>
		
		<pre class="file" id="file57" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
        "time"

        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/cache"
        "volcano.sh/volcano/pkg/scheduler/conf"
        "volcano.sh/volcano/pkg/scheduler/metrics"
)

// OpenSession start the session
func OpenSession(cache cache.Cache, tiers []conf.Tier, configurations []conf.Configuration) *Session <span class="cov0" title="0">{
        ssn := openSession(cache)
        ssn.Tiers = tiers
        ssn.Configurations = configurations

        for _, tier := range tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if pb, found := GetPluginBuilder(plugin.Name); !found </span><span class="cov0" title="0">{
                                klog.Errorf("Failed to get plugin %s.", plugin.Name)
                        }</span> else<span class="cov0" title="0"> {
                                plugin := pb(plugin.Arguments)
                                ssn.plugins[plugin.Name()] = plugin
                                onSessionOpenStart := time.Now()
                                plugin.OnSessionOpen(ssn)
                                metrics.UpdatePluginDuration(plugin.Name(), metrics.OnSessionOpen, metrics.Duration(onSessionOpenStart))
                        }</span>
                }
        }
        <span class="cov0" title="0">return ssn</span>
}

// CloseSession close the session
func CloseSession(ssn *Session) <span class="cov0" title="0">{
        for _, plugin := range ssn.plugins </span><span class="cov0" title="0">{
                onSessionCloseStart := time.Now()
                plugin.OnSessionClose(ssn)
                metrics.UpdatePluginDuration(plugin.Name(), metrics.OnSessionClose, metrics.Duration(onSessionCloseStart))
        }</span>

        <span class="cov0" title="0">closeSession(ssn)</span>
}
</pre>
		
		<pre class="file" id="file58" style="display: none">package framework

import (
        "context"
        "math/rand"
        "reflect"
        "time"

        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/scheduling"
        "volcano.sh/volcano/pkg/scheduler/api"
)

const (
        jobUpdaterWorker = 16

        jobConditionUpdateTime       = time.Minute
        jobConditionUpdateTimeJitter = 30 * time.Second
)

// TimeJitterAfter means: new after old + duration + jitter
func TimeJitterAfter(new, old time.Time, duration, maxJitter time.Duration) bool <span class="cov0" title="0">{
        var jitter int64
        if maxJitter &gt; 0 </span><span class="cov0" title="0">{
                jitter = rand.Int63n(int64(maxJitter))
        }</span>
        <span class="cov0" title="0">return new.After(old.Add(duration + time.Duration(jitter)))</span>
}

type jobUpdater struct {
        ssn      *Session
        jobQueue []*api.JobInfo
}

func newJobUpdater(ssn *Session) *jobUpdater <span class="cov0" title="0">{
        queue := make([]*api.JobInfo, 0, len(ssn.Jobs))
        for _, job := range ssn.Jobs </span><span class="cov0" title="0">{
                queue = append(queue, job)
        }</span>

        <span class="cov0" title="0">ju := &amp;jobUpdater{
                ssn:      ssn,
                jobQueue: queue,
        }
        return ju</span>
}

func (ju *jobUpdater) UpdateAll() <span class="cov0" title="0">{
        workqueue.ParallelizeUntil(context.TODO(), jobUpdaterWorker, len(ju.jobQueue), ju.updateJob)
}</span>

func isPodGroupConditionsUpdated(newCondition, oldCondition []scheduling.PodGroupCondition) bool <span class="cov0" title="0">{
        if len(newCondition) != len(oldCondition) </span><span class="cov0" title="0">{
                return true
        }</span>

        <span class="cov0" title="0">for index, newCond := range newCondition </span><span class="cov0" title="0">{
                oldCond := oldCondition[index]

                newTime := newCond.LastTransitionTime
                oldTime := oldCond.LastTransitionTime
                if TimeJitterAfter(newTime.Time, oldTime.Time, jobConditionUpdateTime, jobConditionUpdateTimeJitter) </span><span class="cov0" title="0">{
                        return true
                }</span>

                // if newCond is not new enough, we treat it the same as the old one
                <span class="cov0" title="0">newCond.LastTransitionTime = oldTime

                // comparing should ignore the TransitionID
                newTransitionID := newCond.TransitionID
                newCond.TransitionID = oldCond.TransitionID

                shouldUpdate := !reflect.DeepEqual(&amp;newCond, &amp;oldCond)

                newCond.LastTransitionTime = newTime
                newCond.TransitionID = newTransitionID
                if shouldUpdate </span><span class="cov0" title="0">{
                        return true
                }</span>
        }

        <span class="cov0" title="0">return false</span>
}

func isPodGroupStatusUpdated(newStatus, oldStatus scheduling.PodGroupStatus) bool <span class="cov0" title="0">{
        newCondition := newStatus.Conditions
        newStatus.Conditions = nil
        oldCondition := oldStatus.Conditions
        oldStatus.Conditions = nil

        return !reflect.DeepEqual(newStatus, oldStatus) || isPodGroupConditionsUpdated(newCondition, oldCondition)
}</span>

// updateJob update specified job
func (ju *jobUpdater) updateJob(index int) <span class="cov0" title="0">{
        job := ju.jobQueue[index]
        ssn := ju.ssn

        job.PodGroup.Status = jobStatus(ssn, job)
        oldStatus, found := ssn.podGroupStatus[job.UID]
        updatePG := !found || isPodGroupStatusUpdated(job.PodGroup.Status, oldStatus)
        if _, err := ssn.cache.UpdateJobStatus(job, updatePG); err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Failed to update job &lt;%s/%s&gt;: %v",
                        job.Namespace, job.Name, err)
        }</span>
}
</pre>
		
		<pre class="file" id="file59" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
        "fmt"
        "path/filepath"
        "plugin"
        "strings"
        "sync"

        "k8s.io/klog"
)

var pluginMutex sync.Mutex

// PluginBuilder plugin management
type PluginBuilder = func(Arguments) Plugin

// Plugin management
var pluginBuilders = map[string]PluginBuilder{}

// RegisterPluginBuilder register the plugin
func RegisterPluginBuilder(name string, pc PluginBuilder) <span class="cov0" title="0">{
        pluginMutex.Lock()
        defer pluginMutex.Unlock()

        pluginBuilders[name] = pc
}</span>

// CleanupPluginBuilders cleans up all the plugin
func CleanupPluginBuilders() <span class="cov0" title="0">{
        pluginMutex.Lock()
        defer pluginMutex.Unlock()

        pluginBuilders = map[string]PluginBuilder{}
}</span>

// GetPluginBuilder get the pluginbuilder by name
func GetPluginBuilder(name string) (PluginBuilder, bool) <span class="cov0" title="0">{
        pluginMutex.Lock()
        defer pluginMutex.Unlock()

        pb, found := pluginBuilders[name]
        return pb, found
}</span>

// LoadCustomPlugins loads custom implement plugins
func LoadCustomPlugins(pluginsDir string) error <span class="cov0" title="0">{
        pluginPaths, _ := filepath.Glob(fmt.Sprintf("%s/*.so", pluginsDir))
        for _, pluginPath := range pluginPaths </span><span class="cov0" title="0">{
                pluginBuilder, err := loadPluginBuilder(pluginPath)
                if err != nil </span><span class="cov0" title="0">{
                        return err
                }</span>
                <span class="cov0" title="0">pluginName := getPluginName(pluginPath)
                RegisterPluginBuilder(pluginName, pluginBuilder)
                klog.V(4).Infof("Custom plugin %s loaded", pluginName)</span>
        }

        <span class="cov0" title="0">return nil</span>
}

func getPluginName(pluginPath string) string <span class="cov8" title="1">{
        return strings.TrimSuffix(filepath.Base(pluginPath), filepath.Ext(pluginPath))
}</span>

func loadPluginBuilder(pluginPath string) (PluginBuilder, error) <span class="cov0" title="0">{
        plug, err := plugin.Open(pluginPath)
        if err != nil </span><span class="cov0" title="0">{
                return nil, err
        }</span>

        <span class="cov0" title="0">symBuilder, err := plug.Lookup("New")
        if err != nil </span><span class="cov0" title="0">{
                return nil, err
        }</span>

        <span class="cov0" title="0">builder, ok := symBuilder.(PluginBuilder)
        if !ok </span><span class="cov0" title="0">{
                return nil, fmt.Errorf("unexpected plugin: %s, failed to convert PluginBuilder `New`", pluginPath)
        }</span>

        <span class="cov0" title="0">return builder, nil</span>
}

// Action management
var actionMap = map[string]Action{}

// RegisterAction register action
func RegisterAction(act Action) <span class="cov0" title="0">{
        pluginMutex.Lock()
        defer pluginMutex.Unlock()

        actionMap[act.Name()] = act
}</span>

// GetAction get the action by name
func GetAction(name string) (Action, bool) <span class="cov0" title="0">{
        pluginMutex.Lock()
        defer pluginMutex.Unlock()

        act, found := actionMap[name]
        return act, found
}</span>
</pre>
		
		<pre class="file" id="file60" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
        "fmt"

        v1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/apimachinery/pkg/util/uuid"
        "k8s.io/client-go/informers"
        "k8s.io/client-go/kubernetes"
        "k8s.io/klog"
        volumescheduling "k8s.io/kubernetes/pkg/controller/volume/scheduling"

        "volcano.sh/apis/pkg/apis/scheduling"
        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/cache"
        "volcano.sh/volcano/pkg/scheduler/conf"
        "volcano.sh/volcano/pkg/scheduler/metrics"
        "volcano.sh/volcano/pkg/scheduler/util"
)

// Session information for the current session
type Session struct {
        UID types.UID

        kubeClient      kubernetes.Interface
        cache           cache.Cache
        informerFactory informers.SharedInformerFactory

        TotalResource *api.Resource
        // podGroupStatus cache podgroup status during schedule
        // This should not be mutated after initiated
        podGroupStatus map[api.JobID]scheduling.PodGroupStatus

        Jobs           map[api.JobID]*api.JobInfo
        Nodes          map[string]*api.NodeInfo
        RevocableNodes map[string]*api.NodeInfo
        Queues         map[api.QueueID]*api.QueueInfo
        NamespaceInfo  map[api.NamespaceName]*api.NamespaceInfo

        Tiers          []conf.Tier
        Configurations []conf.Configuration
        NodeList       []*api.NodeInfo

        plugins           map[string]Plugin
        eventHandlers     []*EventHandler
        jobOrderFns       map[string]api.CompareFn
        queueOrderFns     map[string]api.CompareFn
        taskOrderFns      map[string]api.CompareFn
        namespaceOrderFns map[string]api.CompareFn
        clusterOrderFns   map[string]api.CompareFn
        predicateFns      map[string]api.PredicateFn
        bestNodeFns       map[string]api.BestNodeFn
        nodeOrderFns      map[string]api.NodeOrderFn
        batchNodeOrderFns map[string]api.BatchNodeOrderFn
        nodeMapFns        map[string]api.NodeMapFn
        nodeReduceFns     map[string]api.NodeReduceFn
        preemptableFns    map[string]api.EvictableFn
        reclaimableFns    map[string]api.EvictableFn
        overusedFns       map[string]api.ValidateFn
        underUsedFns      map[string]api.UnderUsedResourceFn
        jobReadyFns       map[string]api.ValidateFn
        jobPipelinedFns   map[string]api.VoteFn
        jobValidFns       map[string]api.ValidateExFn
        jobEnqueueableFns map[string]api.VoteFn
        jobEnqueuedFns    map[string]api.JobEnqueuedFn
        targetJobFns      map[string]api.TargetJobFn
        reservedNodesFns  map[string]api.ReservedNodesFn
        victimTasksFns    map[string]api.VictimTasksFn
        jobStarvingFns    map[string]api.ValidateFn
}

func openSession(cache cache.Cache) *Session <span class="cov0" title="0">{
        ssn := &amp;Session{
                UID:             uuid.NewUUID(),
                kubeClient:      cache.Client(),
                cache:           cache,
                informerFactory: cache.SharedInformerFactory(),

                TotalResource:  api.EmptyResource(),
                podGroupStatus: map[api.JobID]scheduling.PodGroupStatus{},

                Jobs:           map[api.JobID]*api.JobInfo{},
                Nodes:          map[string]*api.NodeInfo{},
                RevocableNodes: map[string]*api.NodeInfo{},
                Queues:         map[api.QueueID]*api.QueueInfo{},

                plugins:           map[string]Plugin{},
                jobOrderFns:       map[string]api.CompareFn{},
                queueOrderFns:     map[string]api.CompareFn{},
                taskOrderFns:      map[string]api.CompareFn{},
                namespaceOrderFns: map[string]api.CompareFn{},
                clusterOrderFns:   map[string]api.CompareFn{},
                predicateFns:      map[string]api.PredicateFn{},
                bestNodeFns:       map[string]api.BestNodeFn{},
                nodeOrderFns:      map[string]api.NodeOrderFn{},
                batchNodeOrderFns: map[string]api.BatchNodeOrderFn{},
                nodeMapFns:        map[string]api.NodeMapFn{},
                nodeReduceFns:     map[string]api.NodeReduceFn{},
                preemptableFns:    map[string]api.EvictableFn{},
                reclaimableFns:    map[string]api.EvictableFn{},
                overusedFns:       map[string]api.ValidateFn{},
                underUsedFns:      map[string]api.UnderUsedResourceFn{},
                jobReadyFns:       map[string]api.ValidateFn{},
                jobPipelinedFns:   map[string]api.VoteFn{},
                jobValidFns:       map[string]api.ValidateExFn{},
                jobEnqueueableFns: map[string]api.VoteFn{},
                jobEnqueuedFns:    map[string]api.JobEnqueuedFn{},
                targetJobFns:      map[string]api.TargetJobFn{},
                reservedNodesFns:  map[string]api.ReservedNodesFn{},
                victimTasksFns:    map[string]api.VictimTasksFn{},
                jobStarvingFns:    map[string]api.ValidateFn{},
        }

        snapshot := cache.Snapshot()

        ssn.Jobs = snapshot.Jobs
        for _, job := range ssn.Jobs </span><span class="cov0" title="0">{
                // only conditions will be updated periodically
                if job.PodGroup != nil &amp;&amp; job.PodGroup.Status.Conditions != nil </span><span class="cov0" title="0">{
                        ssn.podGroupStatus[job.UID] = job.PodGroup.Status
                }</span>

                <span class="cov0" title="0">if vjr := ssn.JobValid(job); vjr != nil </span><span class="cov0" title="0">{
                        if !vjr.Pass </span><span class="cov0" title="0">{
                                jc := &amp;scheduling.PodGroupCondition{
                                        Type:               scheduling.PodGroupUnschedulableType,
                                        Status:             v1.ConditionTrue,
                                        LastTransitionTime: metav1.Now(),
                                        TransitionID:       string(ssn.UID),
                                        Reason:             vjr.Reason,
                                        Message:            vjr.Message,
                                }

                                if err := ssn.UpdatePodGroupCondition(job, jc); err != nil </span><span class="cov0" title="0">{
                                        klog.Errorf("Failed to update job condition: %v", err)
                                }</span>
                        }

                        <span class="cov0" title="0">delete(ssn.Jobs, job.UID)</span>
                }
        }
        <span class="cov0" title="0">ssn.NodeList = util.GetNodeList(snapshot.Nodes, snapshot.NodeList)
        ssn.Nodes = snapshot.Nodes
        ssn.RevocableNodes = snapshot.RevocableNodes
        ssn.Queues = snapshot.Queues
        ssn.NamespaceInfo = snapshot.NamespaceInfo
        // calculate all nodes' resource only once in each schedule cycle, other plugins can clone it when need
        for _, n := range ssn.Nodes </span><span class="cov0" title="0">{
                ssn.TotalResource.Add(n.Allocatable)
        }</span>

        <span class="cov0" title="0">klog.V(3).Infof("Open Session %v with &lt;%d&gt; Job and &lt;%d&gt; Queues",
                ssn.UID, len(ssn.Jobs), len(ssn.Queues))

        return ssn</span>
}

func closeSession(ssn *Session) <span class="cov0" title="0">{
        ju := newJobUpdater(ssn)
        ju.UpdateAll()

        ssn.Jobs = nil
        ssn.Nodes = nil
        ssn.RevocableNodes = nil
        ssn.plugins = nil
        ssn.eventHandlers = nil
        ssn.jobOrderFns = nil
        ssn.namespaceOrderFns = nil
        ssn.queueOrderFns = nil
        ssn.clusterOrderFns = nil
        ssn.NodeList = nil
        ssn.TotalResource = nil

        klog.V(3).Infof("Close Session %v", ssn.UID)
}</span>

func jobStatus(ssn *Session, jobInfo *api.JobInfo) scheduling.PodGroupStatus <span class="cov0" title="0">{
        status := jobInfo.PodGroup.Status

        unschedulable := false
        for _, c := range status.Conditions </span><span class="cov0" title="0">{
                if c.Type == scheduling.PodGroupUnschedulableType &amp;&amp;
                        c.Status == v1.ConditionTrue &amp;&amp;
                        c.TransitionID == string(ssn.UID) </span><span class="cov0" title="0">{
                        unschedulable = true
                        break</span>
                }
        }

        // If running tasks &amp;&amp; unschedulable, unknown phase
        <span class="cov0" title="0">if len(jobInfo.TaskStatusIndex[api.Running]) != 0 &amp;&amp; unschedulable </span><span class="cov0" title="0">{
                status.Phase = scheduling.PodGroupUnknown
        }</span> else<span class="cov0" title="0"> {
                allocated := 0
                for status, tasks := range jobInfo.TaskStatusIndex </span><span class="cov0" title="0">{
                        if api.AllocatedStatus(status) || status == api.Succeeded </span><span class="cov0" title="0">{
                                allocated += len(tasks)
                        }</span>
                }

                // If there're enough allocated resource, it's running
                <span class="cov0" title="0">if int32(allocated) &gt;= jobInfo.PodGroup.Spec.MinMember </span><span class="cov0" title="0">{
                        status.Phase = scheduling.PodGroupRunning
                }</span> else<span class="cov0" title="0"> if jobInfo.PodGroup.Status.Phase != scheduling.PodGroupInqueue </span><span class="cov0" title="0">{
                        status.Phase = scheduling.PodGroupPending
                }</span>
        }

        <span class="cov0" title="0">status.Running = int32(len(jobInfo.TaskStatusIndex[api.Running]))
        status.Failed = int32(len(jobInfo.TaskStatusIndex[api.Failed]))
        status.Succeeded = int32(len(jobInfo.TaskStatusIndex[api.Succeeded]))

        return status</span>
}

// Statement returns new statement object
func (ssn *Session) Statement() *Statement <span class="cov0" title="0">{
        return &amp;Statement{
                ssn: ssn,
        }
}</span>

// Pipeline  the task to the node in the session
func (ssn *Session) Pipeline(task *api.TaskInfo, hostname string) error <span class="cov0" title="0">{
        // Only update status in session
        job, found := ssn.Jobs[task.Job]
        if found </span><span class="cov0" title="0">{
                if err := job.UpdateTaskStatus(task, api.Pipelined); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; status to %v in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, api.Pipelined, ssn.UID, err)
                        return err
                }</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Job &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        task.Job, ssn.UID)
                return fmt.Errorf("failed to find job %s when binding", task.Job)
        }</span>

        <span class="cov0" title="0">task.NodeName = hostname

        if node, found := ssn.Nodes[hostname]; found </span><span class="cov0" title="0">{
                if err := node.AddTask(task); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to add task &lt;%v/%v&gt; to node &lt;%v&gt; in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, hostname, ssn.UID, err)
                        return err
                }</span>
                <span class="cov0" title="0">klog.V(3).Infof("After added Task &lt;%v/%v&gt; to Node &lt;%v&gt;: idle &lt;%v&gt;, used &lt;%v&gt;, releasing &lt;%v&gt;",
                        task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing)</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Node &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        hostname, ssn.UID)
                return fmt.Errorf("failed to find node %s", hostname)
        }</span>

        <span class="cov0" title="0">for _, eh := range ssn.eventHandlers </span><span class="cov0" title="0">{
                if eh.AllocateFunc != nil </span><span class="cov0" title="0">{
                        eh.AllocateFunc(&amp;Event{
                                Task: task,
                        })
                }</span>
        }

        <span class="cov0" title="0">return nil</span>
}

//Allocate the task to the node in the session
func (ssn *Session) Allocate(task *api.TaskInfo, nodeInfo *api.NodeInfo) error <span class="cov0" title="0">{
        podVolumes, err := ssn.cache.GetPodVolumes(task, nodeInfo.Node)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">hostname := nodeInfo.Name
        if err := ssn.cache.AllocateVolumes(task, hostname, podVolumes); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">task.Pod.Spec.NodeName = hostname
        task.PodVolumes = podVolumes

        // Only update status in session
        job, found := ssn.Jobs[task.Job]
        if found </span><span class="cov0" title="0">{
                if err := job.UpdateTaskStatus(task, api.Allocated); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; status to %v in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, api.Allocated, ssn.UID, err)
                        return err
                }</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Job &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        task.Job, ssn.UID)
                return fmt.Errorf("failed to find job %s", task.Job)
        }</span>

        <span class="cov0" title="0">task.NodeName = hostname

        if node, found := ssn.Nodes[hostname]; found </span><span class="cov0" title="0">{
                if err := node.AddTask(task); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to add task &lt;%v/%v&gt; to node &lt;%v&gt; in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, hostname, ssn.UID, err)
                        return err
                }</span>
                <span class="cov0" title="0">klog.V(3).Infof("After allocated Task &lt;%v/%v&gt; to Node &lt;%v&gt;: idle &lt;%v&gt;, used &lt;%v&gt;, releasing &lt;%v&gt;",
                        task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing)</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Node &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        hostname, ssn.UID)
                return fmt.Errorf("failed to find node %s", hostname)
        }</span>

        // Callbacks
        <span class="cov0" title="0">for _, eh := range ssn.eventHandlers </span><span class="cov0" title="0">{
                if eh.AllocateFunc != nil </span><span class="cov0" title="0">{
                        eh.AllocateFunc(&amp;Event{
                                Task: task,
                        })
                }</span>
        }

        <span class="cov0" title="0">if ssn.JobReady(job) </span><span class="cov0" title="0">{
                for _, task := range job.TaskStatusIndex[api.Allocated] </span><span class="cov0" title="0">{
                        if err := ssn.dispatch(task, podVolumes); err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("Failed to dispatch task &lt;%v/%v&gt;: %v",
                                        task.Namespace, task.Name, err)
                                return err
                        }</span>
                }
        }

        <span class="cov0" title="0">return nil</span>
}

func (ssn *Session) dispatch(task *api.TaskInfo, volumes *volumescheduling.PodVolumes) error <span class="cov0" title="0">{
        if err := ssn.cache.BindVolumes(task, volumes); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">if err := ssn.cache.Bind(task, task.NodeName); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        // Update status in session
        <span class="cov0" title="0">if job, found := ssn.Jobs[task.Job]; found </span><span class="cov0" title="0">{
                if err := job.UpdateTaskStatus(task, api.Binding); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; status to %v in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, api.Binding, ssn.UID, err)
                        return err
                }</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Job &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        task.Job, ssn.UID)
                return fmt.Errorf("failed to find job %s", task.Job)
        }</span>

        <span class="cov0" title="0">metrics.UpdateTaskScheduleDuration(metrics.Duration(task.Pod.CreationTimestamp.Time))
        return nil</span>
}

//Evict the task in the session
func (ssn *Session) Evict(reclaimee *api.TaskInfo, reason string) error <span class="cov0" title="0">{
        if err := ssn.cache.Evict(reclaimee, reason); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        // Update status in session
        <span class="cov0" title="0">job, found := ssn.Jobs[reclaimee.Job]
        if found </span><span class="cov0" title="0">{
                if err := job.UpdateTaskStatus(reclaimee, api.Releasing); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; status to %v in Session &lt;%v&gt;: %v",
                                reclaimee.Namespace, reclaimee.Name, api.Releasing, ssn.UID, err)
                        return err
                }</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Job &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        reclaimee.Job, ssn.UID)
                return fmt.Errorf("failed to find job %s", reclaimee.Job)
        }</span>

        // Update task in node.
        <span class="cov0" title="0">if node, found := ssn.Nodes[reclaimee.NodeName]; found </span><span class="cov0" title="0">{
                if err := node.UpdateTask(reclaimee); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; in Session &lt;%v&gt;: %v",
                                reclaimee.Namespace, reclaimee.Name, ssn.UID, err)
                        return err
                }</span>
        }

        <span class="cov0" title="0">for _, eh := range ssn.eventHandlers </span><span class="cov0" title="0">{
                if eh.DeallocateFunc != nil </span><span class="cov0" title="0">{
                        eh.DeallocateFunc(&amp;Event{
                                Task: reclaimee,
                        })
                }</span>
        }

        <span class="cov0" title="0">return nil</span>
}

// BindPodGroup bind PodGroup to specified cluster
func (ssn *Session) BindPodGroup(job *api.JobInfo, cluster string) error <span class="cov0" title="0">{
        return ssn.cache.BindPodGroup(job, cluster)
}</span>

// UpdatePodGroupCondition update job condition accordingly.
func (ssn *Session) UpdatePodGroupCondition(jobInfo *api.JobInfo, cond *scheduling.PodGroupCondition) error <span class="cov0" title="0">{
        job, ok := ssn.Jobs[jobInfo.UID]
        if !ok </span><span class="cov0" title="0">{
                return fmt.Errorf("failed to find job &lt;%s/%s&gt;", jobInfo.Namespace, jobInfo.Name)
        }</span>

        <span class="cov0" title="0">index := -1
        for i, c := range job.PodGroup.Status.Conditions </span><span class="cov0" title="0">{
                if c.Type == cond.Type </span><span class="cov0" title="0">{
                        index = i
                        break</span>
                }
        }

        // Update condition to the new condition.
        <span class="cov0" title="0">if index &lt; 0 </span><span class="cov0" title="0">{
                job.PodGroup.Status.Conditions = append(job.PodGroup.Status.Conditions, *cond)
        }</span> else<span class="cov0" title="0"> {
                job.PodGroup.Status.Conditions[index] = *cond
        }</span>

        <span class="cov0" title="0">return nil</span>
}

// AddEventHandler add event handlers
func (ssn *Session) AddEventHandler(eh *EventHandler) <span class="cov0" title="0">{
        ssn.eventHandlers = append(ssn.eventHandlers, eh)
}</span>

// UpdateSchedulerNumaInfo update SchedulerNumaInfo
func (ssn *Session) UpdateSchedulerNumaInfo(AllocatedSets map[string]api.ResNumaSets) <span class="cov0" title="0">{
        ssn.cache.UpdateSchedulerNumaInfo(AllocatedSets)
}</span>

// KubeClient returns the kubernetes client
func (ssn Session) KubeClient() kubernetes.Interface <span class="cov0" title="0">{
        return ssn.kubeClient
}</span>

// InformerFactory returns the scheduler ShareInformerFactory
func (ssn Session) InformerFactory() informers.SharedInformerFactory <span class="cov0" title="0">{
        return ssn.informerFactory
}</span>

//String return nodes and jobs information in the session
func (ssn Session) String() string <span class="cov0" title="0">{
        msg := fmt.Sprintf("Session %v: \n", ssn.UID)

        for _, job := range ssn.Jobs </span><span class="cov0" title="0">{
                msg = fmt.Sprintf("%s%v\n", msg, job)
        }</span>

        <span class="cov0" title="0">for _, node := range ssn.Nodes </span><span class="cov0" title="0">{
                msg = fmt.Sprintf("%s%v\n", msg, node)
        }</span>

        <span class="cov0" title="0">return msg</span>
}
</pre>
		
		<pre class="file" id="file61" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
        k8sframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"

        "volcano.sh/apis/pkg/apis/scheduling"
        "volcano.sh/volcano/pkg/scheduler/api"
)

// AddJobOrderFn add job order function
func (ssn *Session) AddJobOrderFn(name string, cf api.CompareFn) <span class="cov0" title="0">{
        ssn.jobOrderFns[name] = cf
}</span>

// AddQueueOrderFn add queue order function
func (ssn *Session) AddQueueOrderFn(name string, qf api.CompareFn) <span class="cov0" title="0">{
        ssn.queueOrderFns[name] = qf
}</span>

// AddClusterOrderFn add queue order function
func (ssn *Session) AddClusterOrderFn(name string, qf api.CompareFn) <span class="cov0" title="0">{
        ssn.clusterOrderFns[name] = qf
}</span>

// AddTaskOrderFn add task order function
func (ssn *Session) AddTaskOrderFn(name string, cf api.CompareFn) <span class="cov0" title="0">{
        ssn.taskOrderFns[name] = cf
}</span>

// AddNamespaceOrderFn add namespace order function
func (ssn *Session) AddNamespaceOrderFn(name string, cf api.CompareFn) <span class="cov0" title="0">{
        ssn.namespaceOrderFns[name] = cf
}</span>

// AddPreemptableFn add preemptable function
func (ssn *Session) AddPreemptableFn(name string, cf api.EvictableFn) <span class="cov0" title="0">{
        ssn.preemptableFns[name] = cf
}</span>

// AddReclaimableFn add Reclaimable function
func (ssn *Session) AddReclaimableFn(name string, rf api.EvictableFn) <span class="cov0" title="0">{
        ssn.reclaimableFns[name] = rf
}</span>

// AddJobReadyFn add JobReady function
func (ssn *Session) AddJobReadyFn(name string, vf api.ValidateFn) <span class="cov0" title="0">{
        ssn.jobReadyFns[name] = vf
}</span>

// AddJobPipelinedFn add pipelined function
func (ssn *Session) AddJobPipelinedFn(name string, vf api.VoteFn) <span class="cov0" title="0">{
        ssn.jobPipelinedFns[name] = vf
}</span>

// AddPredicateFn add Predicate function
func (ssn *Session) AddPredicateFn(name string, pf api.PredicateFn) <span class="cov0" title="0">{
        ssn.predicateFns[name] = pf
}</span>

// AddBestNodeFn add BestNode function
func (ssn *Session) AddBestNodeFn(name string, pf api.BestNodeFn) <span class="cov0" title="0">{
        ssn.bestNodeFns[name] = pf
}</span>

// AddNodeOrderFn add Node order function
func (ssn *Session) AddNodeOrderFn(name string, pf api.NodeOrderFn) <span class="cov0" title="0">{
        ssn.nodeOrderFns[name] = pf
}</span>

// AddBatchNodeOrderFn add Batch Node order function
func (ssn *Session) AddBatchNodeOrderFn(name string, pf api.BatchNodeOrderFn) <span class="cov0" title="0">{
        ssn.batchNodeOrderFns[name] = pf
}</span>

// AddNodeMapFn add Node map function
func (ssn *Session) AddNodeMapFn(name string, pf api.NodeMapFn) <span class="cov0" title="0">{
        ssn.nodeMapFns[name] = pf
}</span>

// AddNodeReduceFn add Node reduce function
func (ssn *Session) AddNodeReduceFn(name string, pf api.NodeReduceFn) <span class="cov0" title="0">{
        ssn.nodeReduceFns[name] = pf
}</span>

// AddOverusedFn add overused function
func (ssn *Session) AddOverusedFn(name string, fn api.ValidateFn) <span class="cov0" title="0">{
        ssn.overusedFns[name] = fn
}</span>

// AddUnderusedResourceFn add underused function
func (ssn *Session) AddUnderusedResourceFn(name string, fn api.UnderUsedResourceFn) <span class="cov0" title="0">{
        ssn.underUsedFns[name] = fn
}</span>

// AddJobValidFn add jobvalid function
func (ssn *Session) AddJobValidFn(name string, fn api.ValidateExFn) <span class="cov0" title="0">{
        ssn.jobValidFns[name] = fn
}</span>

// AddJobEnqueueableFn add jobenqueueable function
func (ssn *Session) AddJobEnqueueableFn(name string, fn api.VoteFn) <span class="cov0" title="0">{
        ssn.jobEnqueueableFns[name] = fn
}</span>

// AddJobEnqueuedFn add jobEnqueued function
func (ssn *Session) AddJobEnqueuedFn(name string, fn api.JobEnqueuedFn) <span class="cov0" title="0">{
        ssn.jobEnqueuedFns[name] = fn
}</span>

// AddTargetJobFn add targetjob function
func (ssn *Session) AddTargetJobFn(name string, fn api.TargetJobFn) <span class="cov0" title="0">{
        ssn.targetJobFns[name] = fn
}</span>

// AddReservedNodesFn add reservedNodesFn function
func (ssn *Session) AddReservedNodesFn(name string, fn api.ReservedNodesFn) <span class="cov0" title="0">{
        ssn.reservedNodesFns[name] = fn
}</span>

// AddVictimTasksFns add victimTasksFns function
func (ssn *Session) AddVictimTasksFns(name string, fn api.VictimTasksFn) <span class="cov0" title="0">{
        ssn.victimTasksFns[name] = fn
}</span>

// AddJobStarvingFns add jobStarvingFns function
func (ssn *Session) AddJobStarvingFns(name string, fn api.ValidateFn) <span class="cov0" title="0">{
        ssn.jobStarvingFns[name] = fn
}</span>

// Reclaimable invoke reclaimable function of the plugins
func (ssn *Session) Reclaimable(reclaimer *api.TaskInfo, reclaimees []*api.TaskInfo) []*api.TaskInfo <span class="cov0" title="0">{
        var victims []*api.TaskInfo
        var init bool

        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledReclaimable) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">rf, found := ssn.reclaimableFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov0" title="0">candidates, abstain := rf(reclaimer, reclaimees)
                        if abstain == 0 </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">if len(candidates) == 0 </span><span class="cov0" title="0">{
                                victims = nil
                                break</span>
                        }
                        <span class="cov0" title="0">if !init </span><span class="cov0" title="0">{
                                victims = candidates
                                init = true
                        }</span> else<span class="cov0" title="0"> {
                                var intersection []*api.TaskInfo
                                // Get intersection of victims and candidates.
                                for _, v := range victims </span><span class="cov0" title="0">{
                                        for _, c := range candidates </span><span class="cov0" title="0">{
                                                if v.UID == c.UID </span><span class="cov0" title="0">{
                                                        intersection = append(intersection, v)
                                                }</span>
                                        }
                                }

                                // Update victims to intersection
                                <span class="cov0" title="0">victims = intersection</span>
                        }
                }
                // Plugins in this tier made decision if victims is not nil
                <span class="cov0" title="0">if victims != nil </span><span class="cov0" title="0">{
                        return victims
                }</span>
        }

        <span class="cov0" title="0">return victims</span>
}

// Preemptable invoke preemptable function of the plugins
func (ssn *Session) Preemptable(preemptor *api.TaskInfo, preemptees []*api.TaskInfo) []*api.TaskInfo <span class="cov0" title="0">{
        var victims []*api.TaskInfo
        var init bool

        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledPreemptable) </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov0" title="0">pf, found := ssn.preemptableFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">candidates, abstain := pf(preemptor, preemptees)
                        if abstain == 0 </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        // intersection will be nil if length is 0, don't need to do any more check
                        <span class="cov0" title="0">if len(candidates) == 0 </span><span class="cov0" title="0">{
                                victims = nil
                                break</span>
                        }

                        <span class="cov0" title="0">if !init </span><span class="cov0" title="0">{
                                victims = candidates
                                init = true
                        }</span> else<span class="cov0" title="0"> {
                                var intersection []*api.TaskInfo
                                // Get intersection of victims and candidates.
                                for _, v := range victims </span><span class="cov0" title="0">{
                                        for _, c := range candidates </span><span class="cov0" title="0">{
                                                if v.UID == c.UID </span><span class="cov0" title="0">{
                                                        intersection = append(intersection, v)
                                                }</span>
                                        }
                                }

                                // Update victims to intersection
                                <span class="cov0" title="0">victims = intersection</span>
                        }
                }
                // Plugins in this tier made decision if victims is not nil
                <span class="cov0" title="0">if victims != nil </span><span class="cov0" title="0">{
                        return victims
                }</span>
        }

        <span class="cov0" title="0">return victims</span>
}

// Overused invoke overused function of the plugins
func (ssn *Session) Overused(queue *api.QueueInfo) bool <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        of, found := ssn.overusedFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">if of(queue) </span><span class="cov0" title="0">{
                                return true
                        }</span>
                }
        }

        <span class="cov0" title="0">return false</span>
}

// UnderusedResources invoke underused function of the plugins
// Returns:
//  * nil if no `UnderUsedResourceFn` is registered
//  * [] if no under-used resources
func (ssn *Session) UnderusedResources(queue *api.QueueInfo) api.ResourceNameList <span class="cov0" title="0">{
        if len(ssn.underUsedFns) == 0 </span><span class="cov0" title="0">{
                return nil
        }</span>
        <span class="cov0" title="0">for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        of, found := ssn.underUsedFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">underUsedResourceList := of(queue)
                        return underUsedResourceList</span>
                }
        }

        <span class="cov0" title="0">return api.ResourceNameList{}</span>
}

// JobReady invoke jobready function of the plugins
func (ssn *Session) JobReady(obj interface{}) bool <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledJobReady) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">jrf, found := ssn.jobReadyFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov0" title="0">if !jrf(obj) </span><span class="cov0" title="0">{
                                return false
                        }</span>
                }
        }

        <span class="cov0" title="0">return true</span>
}

// JobPipelined invoke pipelined function of the plugins
// Check if job has get enough resource to run
func (ssn *Session) JobPipelined(obj interface{}) bool <span class="cov0" title="0">{
        var hasFound bool
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledJobPipelined) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">jrf, found := ssn.jobPipelinedFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov0" title="0">res := jrf(obj)
                        if res &lt; 0 </span><span class="cov0" title="0">{
                                return false
                        }</span>
                        <span class="cov0" title="0">if res &gt; 0 </span><span class="cov0" title="0">{
                                hasFound = true
                        }</span>
                }
                // if plugin exists that votes permit, meanwhile other plugin votes abstention,
                // permit job to be pipelined, do not check next tier
                <span class="cov0" title="0">if hasFound </span><span class="cov0" title="0">{
                        return true
                }</span>
        }

        <span class="cov0" title="0">return true</span>
}

// JobStarving invoke jobStarving function of the plugins
// Check if job still need more resource
func (ssn *Session) JobStarving(obj interface{}) bool <span class="cov0" title="0">{
        var hasFound bool
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledJobStarving) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">jrf, found := ssn.jobStarvingFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">hasFound = true

                        if !jrf(obj) </span><span class="cov0" title="0">{
                                return false
                        }</span>
                }
                // this tier registered function
                <span class="cov0" title="0">if hasFound </span><span class="cov0" title="0">{
                        return true
                }</span>
        }

        <span class="cov0" title="0">return false</span>
}

// JobValid invoke jobvalid function of the plugins
func (ssn *Session) JobValid(obj interface{}) *api.ValidateResult <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        jrf, found := ssn.jobValidFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov0" title="0">if vr := jrf(obj); vr != nil &amp;&amp; !vr.Pass </span><span class="cov0" title="0">{
                                return vr
                        }</span>
                }
        }

        <span class="cov0" title="0">return nil</span>
}

// JobEnqueueable invoke jobEnqueueableFns function of the plugins
func (ssn *Session) JobEnqueueable(obj interface{}) bool <span class="cov0" title="0">{
        var hasFound bool
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledJobEnqueued) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">fn, found := ssn.jobEnqueueableFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov0" title="0">res := fn(obj)
                        if res &lt; 0 </span><span class="cov0" title="0">{
                                return false
                        }</span>
                        <span class="cov0" title="0">if res &gt; 0 </span><span class="cov0" title="0">{
                                hasFound = true
                        }</span>
                }
                // if plugin exists that votes permit, meanwhile other plugin votes abstention,
                // permit job to be enqueueable, do not check next tier
                <span class="cov0" title="0">if hasFound </span><span class="cov0" title="0">{
                        return true
                }</span>
        }

        <span class="cov0" title="0">return true</span>
}

// JobEnqueued invoke jobEnqueuedFns function of the plugins
func (ssn *Session) JobEnqueued(obj interface{}) <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledJobEnqueued) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">fn, found := ssn.jobEnqueuedFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov0" title="0">fn(obj)</span>
                }
        }
}

// TargetJob invoke targetJobFns function of the plugins
func (ssn *Session) TargetJob(jobs []*api.JobInfo) *api.JobInfo <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledTargetJob) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">fn, found := ssn.targetJobFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">return fn(jobs)</span>
                }
        }
        <span class="cov0" title="0">return nil</span>
}

// VictimTasks invoke ReservedNodes function of the plugins
func (ssn *Session) VictimTasks() []*api.TaskInfo <span class="cov0" title="0">{
        var victims []*api.TaskInfo
        var init bool

        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledVictim) </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov0" title="0">pf, found := ssn.victimTasksFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">candidates := pf()
                        if !init </span><span class="cov0" title="0">{
                                victims = candidates
                                init = true
                        }</span> else<span class="cov0" title="0"> {
                                var intersection []*api.TaskInfo
                                // Get intersection of victims and candidates.
                                for _, v := range victims </span><span class="cov0" title="0">{
                                        for _, c := range candidates </span><span class="cov0" title="0">{
                                                if v.UID == c.UID </span><span class="cov0" title="0">{
                                                        intersection = append(intersection, v)
                                                }</span>
                                        }
                                }

                                // Update victims to intersection
                                <span class="cov0" title="0">victims = intersection</span>
                        }
                }
                // Plugins in this tier made decision if victims is not nil
                <span class="cov0" title="0">if victims != nil </span><span class="cov0" title="0">{
                        return victims
                }</span>
        }

        <span class="cov0" title="0">return victims</span>
}

// ReservedNodes invoke ReservedNodes function of the plugins
func (ssn *Session) ReservedNodes() <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledReservedNodes) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">fn, found := ssn.reservedNodesFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">fn()</span>
                }
        }
}

// JobOrderFn invoke joborder function of the plugins
func (ssn *Session) JobOrderFn(l, r interface{}) bool <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledJobOrder) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">jof, found := ssn.jobOrderFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">if j := jof(l, r); j != 0 </span><span class="cov0" title="0">{
                                return j &lt; 0
                        }</span>
                }
        }

        // If no job order funcs, order job by CreationTimestamp first, then by UID.
        <span class="cov0" title="0">lv := l.(*api.JobInfo)
        rv := r.(*api.JobInfo)
        if lv.CreationTimestamp.Equal(&amp;rv.CreationTimestamp) </span><span class="cov0" title="0">{
                return lv.UID &lt; rv.UID
        }</span>
        <span class="cov0" title="0">return lv.CreationTimestamp.Before(&amp;rv.CreationTimestamp)</span>
}

// NamespaceOrderFn invoke namespaceorder function of the plugins
func (ssn *Session) NamespaceOrderFn(l, r interface{}) bool <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledNamespaceOrder) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">nof, found := ssn.namespaceOrderFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">if j := nof(l, r); j != 0 </span><span class="cov0" title="0">{
                                return j &lt; 0
                        }</span>
                }
        }

        // TODO(lminzhw): if all NamespaceOrderFn treat these two namespace as the same,
        // we should make the job order have its affect among namespaces.
        // or just schedule namespace one by one
        <span class="cov0" title="0">lv := l.(api.NamespaceName)
        rv := r.(api.NamespaceName)
        return lv &lt; rv</span>
}

// ClusterOrderFn invoke ClusterOrderFn function of the plugins
func (ssn *Session) ClusterOrderFn(l, r interface{}) bool <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledClusterOrder) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">cof, found := ssn.clusterOrderFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">if j := cof(l, r); j != 0 </span><span class="cov0" title="0">{
                                return j &lt; 0
                        }</span>
                }
        }

        // If no cluster order funcs, order cluster by ClusterID
        <span class="cov0" title="0">lv := l.(*scheduling.Cluster)
        rv := r.(*scheduling.Cluster)
        return lv.Name &lt; rv.Name</span>
}

// QueueOrderFn invoke queueorder function of the plugins
func (ssn *Session) QueueOrderFn(l, r interface{}) bool <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledQueueOrder) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">qof, found := ssn.queueOrderFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">if j := qof(l, r); j != 0 </span><span class="cov0" title="0">{
                                return j &lt; 0
                        }</span>
                }
        }

        // If no queue order funcs, order queue by CreationTimestamp first, then by UID.
        <span class="cov0" title="0">lv := l.(*api.QueueInfo)
        rv := r.(*api.QueueInfo)
        if lv.Queue.CreationTimestamp.Equal(&amp;rv.Queue.CreationTimestamp) </span><span class="cov0" title="0">{
                return lv.UID &lt; rv.UID
        }</span>
        <span class="cov0" title="0">return lv.Queue.CreationTimestamp.Before(&amp;rv.Queue.CreationTimestamp)</span>
}

// TaskCompareFns invoke taskorder function of the plugins
func (ssn *Session) TaskCompareFns(l, r interface{}) int <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledTaskOrder) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">tof, found := ssn.taskOrderFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">if j := tof(l, r); j != 0 </span><span class="cov0" title="0">{
                                return j
                        }</span>
                }
        }

        <span class="cov0" title="0">return 0</span>
}

// TaskOrderFn invoke taskorder function of the plugins
func (ssn *Session) TaskOrderFn(l, r interface{}) bool <span class="cov0" title="0">{
        if res := ssn.TaskCompareFns(l, r); res != 0 </span><span class="cov0" title="0">{
                return res &lt; 0
        }</span>

        // If no task order funcs, order task by CreationTimestamp first, then by UID.
        <span class="cov0" title="0">lv := l.(*api.TaskInfo)
        rv := r.(*api.TaskInfo)
        if lv.Pod.CreationTimestamp.Equal(&amp;rv.Pod.CreationTimestamp) </span><span class="cov0" title="0">{
                return lv.UID &lt; rv.UID
        }</span>
        <span class="cov0" title="0">return lv.Pod.CreationTimestamp.Before(&amp;rv.Pod.CreationTimestamp)</span>
}

// PredicateFn invoke predicate function of the plugins
func (ssn *Session) PredicateFn(task *api.TaskInfo, node *api.NodeInfo) error <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledPredicate) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">pfn, found := ssn.predicateFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">err := pfn(task, node)
                        if err != nil </span><span class="cov0" title="0">{
                                return err
                        }</span>
                }
        }
        <span class="cov0" title="0">return nil</span>
}

// BestNodeFn invoke bestNode function of the plugins
func (ssn *Session) BestNodeFn(task *api.TaskInfo, nodeScores map[float64][]*api.NodeInfo) *api.NodeInfo <span class="cov0" title="0">{
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledBestNode) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">pfn, found := ssn.bestNodeFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        // Only the first plugin that enables and realizes bestNodeFn is allowed to choose best node for task
                        <span class="cov0" title="0">if bestNode := pfn(task, nodeScores); bestNode != nil </span><span class="cov0" title="0">{
                                return bestNode
                        }</span>
                }
        }
        <span class="cov0" title="0">return nil</span>
}

// NodeOrderFn invoke node order function of the plugins
func (ssn *Session) NodeOrderFn(task *api.TaskInfo, node *api.NodeInfo) (float64, error) <span class="cov0" title="0">{
        priorityScore := 0.0
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledNodeOrder) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">pfn, found := ssn.nodeOrderFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">score, err := pfn(task, node)
                        if err != nil </span><span class="cov0" title="0">{
                                return 0, err
                        }</span>
                        <span class="cov0" title="0">priorityScore += score</span>
                }
        }
        <span class="cov0" title="0">return priorityScore, nil</span>
}

// BatchNodeOrderFn invoke node order function of the plugins
func (ssn *Session) BatchNodeOrderFn(task *api.TaskInfo, nodes []*api.NodeInfo) (map[string]float64, error) <span class="cov0" title="0">{
        priorityScore := make(map[string]float64, len(nodes))
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledNodeOrder) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">pfn, found := ssn.batchNodeOrderFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">score, err := pfn(task, nodes)
                        if err != nil </span><span class="cov0" title="0">{
                                return nil, err
                        }</span>
                        <span class="cov0" title="0">for nodeName, score := range score </span><span class="cov0" title="0">{
                                priorityScore[nodeName] += score
                        }</span>
                }
        }
        <span class="cov0" title="0">return priorityScore, nil</span>
}

func isEnabled(enabled *bool) bool <span class="cov0" title="0">{
        return enabled != nil &amp;&amp; *enabled
}</span>

// NodeOrderMapFn invoke node order function of the plugins
func (ssn *Session) NodeOrderMapFn(task *api.TaskInfo, node *api.NodeInfo) (map[string]float64, float64, error) <span class="cov0" title="0">{
        nodeScoreMap := map[string]float64{}
        var priorityScore float64
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledNodeOrder) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">if pfn, found := ssn.nodeOrderFns[plugin.Name]; found </span><span class="cov0" title="0">{
                                score, err := pfn(task, node)
                                if err != nil </span><span class="cov0" title="0">{
                                        return nodeScoreMap, priorityScore, err
                                }</span>
                                <span class="cov0" title="0">priorityScore += score</span>
                        }
                        <span class="cov0" title="0">if pfn, found := ssn.nodeMapFns[plugin.Name]; found </span><span class="cov0" title="0">{
                                score, err := pfn(task, node)
                                if err != nil </span><span class="cov0" title="0">{
                                        return nodeScoreMap, priorityScore, err
                                }</span>
                                <span class="cov0" title="0">nodeScoreMap[plugin.Name] = score</span>
                        }
                }
        }
        <span class="cov0" title="0">return nodeScoreMap, priorityScore, nil</span>
}

// NodeOrderReduceFn invoke node order function of the plugins
func (ssn *Session) NodeOrderReduceFn(task *api.TaskInfo, pluginNodeScoreMap map[string]k8sframework.NodeScoreList) (map[string]float64, error) <span class="cov0" title="0">{
        nodeScoreMap := map[string]float64{}
        for _, tier := range ssn.Tiers </span><span class="cov0" title="0">{
                for _, plugin := range tier.Plugins </span><span class="cov0" title="0">{
                        if !isEnabled(plugin.EnabledNodeOrder) </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">pfn, found := ssn.nodeReduceFns[plugin.Name]
                        if !found </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov0" title="0">if err := pfn(task, pluginNodeScoreMap[plugin.Name]); err != nil </span><span class="cov0" title="0">{
                                return nodeScoreMap, err
                        }</span>
                        <span class="cov0" title="0">for _, hp := range pluginNodeScoreMap[plugin.Name] </span><span class="cov0" title="0">{
                                nodeScoreMap[hp.Name] += float64(hp.Score)
                        }</span>
                }
        }
        <span class="cov0" title="0">return nodeScoreMap, nil</span>
}
</pre>
		
		<pre class="file" id="file62" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
        "fmt"

        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/metrics"
)

// Operation type
type Operation int8

const (
        // Evict op
        Evict = iota
        // Pipeline op
        Pipeline
        // Allocate op
        Allocate
)

type operation struct {
        name   Operation
        task   *api.TaskInfo
        reason string
}

// Statement structure
type Statement struct {
        operations []operation
        ssn        *Session
}

// NewStatement returns new statement object
func NewStatement(ssn *Session) *Statement <span class="cov0" title="0">{
        return &amp;Statement{
                ssn: ssn,
        }
}</span>

// Evict the pod
func (s *Statement) Evict(reclaimee *api.TaskInfo, reason string) error <span class="cov0" title="0">{
        // Update status in session
        if job, found := s.ssn.Jobs[reclaimee.Job]; found </span><span class="cov0" title="0">{
                if err := job.UpdateTaskStatus(reclaimee, api.Releasing); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; status to %v in Session &lt;%v&gt;: %v",
                                reclaimee.Namespace, reclaimee.Name, api.Releasing, s.ssn.UID, err)
                }</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Job &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        reclaimee.Job, s.ssn.UID)
        }</span>

        // Update task in node.
        <span class="cov0" title="0">if node, found := s.ssn.Nodes[reclaimee.NodeName]; found </span><span class="cov0" title="0">{
                err := node.UpdateTask(reclaimee)
                if err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; in node %v for: %s",
                                reclaimee.Namespace, reclaimee.Name, reclaimee.NodeName, err.Error())
                        return err
                }</span>
        }

        <span class="cov0" title="0">for _, eh := range s.ssn.eventHandlers </span><span class="cov0" title="0">{
                if eh.DeallocateFunc != nil </span><span class="cov0" title="0">{
                        eh.DeallocateFunc(&amp;Event{
                                Task: reclaimee,
                        })
                }</span>
        }

        <span class="cov0" title="0">s.operations = append(s.operations, operation{
                name:   Evict,
                task:   reclaimee,
                reason: reason,
        })

        return nil</span>
}

func (s *Statement) evict(reclaimee *api.TaskInfo, reason string) error <span class="cov0" title="0">{
        if err := s.ssn.cache.Evict(reclaimee, reason); err != nil </span><span class="cov0" title="0">{
                if e := s.unevict(reclaimee); e != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Faled to unevict task &lt;%v/%v&gt;: %v.",
                                reclaimee.Namespace, reclaimee.Name, e)
                }</span>
                <span class="cov0" title="0">return err</span>
        }

        <span class="cov0" title="0">return nil</span>
}

func (s *Statement) unevict(reclaimee *api.TaskInfo) error <span class="cov0" title="0">{
        // Update status in session
        job, found := s.ssn.Jobs[reclaimee.Job]
        if found </span><span class="cov0" title="0">{
                if err := job.UpdateTaskStatus(reclaimee, api.Running); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; status to %v in Session &lt;%v&gt;: %v",
                                reclaimee.Namespace, reclaimee.Name, api.Releasing, s.ssn.UID, err)
                }</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Job &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        reclaimee.Job, s.ssn.UID)
        }</span>

        // Update task in node.
        <span class="cov0" title="0">if node, found := s.ssn.Nodes[reclaimee.NodeName]; found </span><span class="cov0" title="0">{
                err := node.UpdateTask(reclaimee)
                if err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; in node %v for: %s",
                                reclaimee.Namespace, reclaimee.Name, reclaimee.NodeName, err.Error())
                        return err
                }</span>
        }

        <span class="cov0" title="0">for _, eh := range s.ssn.eventHandlers </span><span class="cov0" title="0">{
                if eh.AllocateFunc != nil </span><span class="cov0" title="0">{
                        eh.AllocateFunc(&amp;Event{
                                Task: reclaimee,
                        })
                }</span>
        }

        <span class="cov0" title="0">return nil</span>
}

// Pipeline the task for the node
func (s *Statement) Pipeline(task *api.TaskInfo, hostname string) error <span class="cov0" title="0">{
        job, found := s.ssn.Jobs[task.Job]
        if found </span><span class="cov0" title="0">{
                if err := job.UpdateTaskStatus(task, api.Pipelined); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; status to %v in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, api.Pipelined, s.ssn.UID, err)
                }</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Job &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        task.Job, s.ssn.UID)
        }</span>

        <span class="cov0" title="0">task.NodeName = hostname

        if node, found := s.ssn.Nodes[hostname]; found </span><span class="cov0" title="0">{
                if err := node.AddTask(task); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to pipeline task &lt;%v/%v&gt; to node &lt;%v&gt; in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, hostname, s.ssn.UID, err)
                }</span>
                <span class="cov0" title="0">klog.V(3).Infof("After pipelined Task &lt;%v/%v&gt; to Node &lt;%v&gt;: idle &lt;%v&gt;, used &lt;%v&gt;, releasing &lt;%v&gt;",
                        task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing)</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Node &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        hostname, s.ssn.UID)
        }</span>

        <span class="cov0" title="0">for _, eh := range s.ssn.eventHandlers </span><span class="cov0" title="0">{
                if eh.AllocateFunc != nil </span><span class="cov0" title="0">{
                        eh.AllocateFunc(&amp;Event{
                                Task: task,
                        })
                }</span>
        }

        <span class="cov0" title="0">s.operations = append(s.operations, operation{
                name: Pipeline,
                task: task,
        })

        return nil</span>
}

func (s *Statement) pipeline(task *api.TaskInfo) {<span class="cov0" title="0">
}</span>

func (s *Statement) unpipeline(task *api.TaskInfo) error <span class="cov0" title="0">{
        job, found := s.ssn.Jobs[task.Job]
        if found </span><span class="cov0" title="0">{
                if err := job.UpdateTaskStatus(task, api.Pending); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; status to %v in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, api.Pipelined, s.ssn.UID, err)
                }</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Job &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        task.Job, s.ssn.UID)
        }</span>

        <span class="cov0" title="0">if node, found := s.ssn.Nodes[task.NodeName]; found </span><span class="cov0" title="0">{
                if err := node.RemoveTask(task); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to pipeline task &lt;%v/%v&gt; to node &lt;%v&gt; in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, task.NodeName, s.ssn.UID, err)
                }</span>
                <span class="cov0" title="0">klog.V(3).Infof("After pipelined Task &lt;%v/%v&gt; to Node &lt;%v&gt;: idle &lt;%v&gt;, used &lt;%v&gt;, releasing &lt;%v&gt;",
                        task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing)</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Node &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        task.NodeName, s.ssn.UID)
        }</span>

        <span class="cov0" title="0">for _, eh := range s.ssn.eventHandlers </span><span class="cov0" title="0">{
                if eh.DeallocateFunc != nil </span><span class="cov0" title="0">{
                        eh.DeallocateFunc(&amp;Event{
                                Task: task,
                        })
                }</span>
        }
        <span class="cov0" title="0">task.NodeName = ""

        return nil</span>
}

// Allocate the task to node
func (s *Statement) Allocate(task *api.TaskInfo, nodeInfo *api.NodeInfo) error <span class="cov0" title="0">{
        podVolumes, err := s.ssn.cache.GetPodVolumes(task, nodeInfo.Node)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">hostname := nodeInfo.Name
        if err := s.ssn.cache.AllocateVolumes(task, hostname, podVolumes); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">task.Pod.Spec.NodeName = hostname
        task.PodVolumes = podVolumes

        // Only update status in session
        job, found := s.ssn.Jobs[task.Job]
        if found </span><span class="cov0" title="0">{
                if err := job.UpdateTaskStatus(task, api.Allocated); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; status to %v in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, api.Allocated, s.ssn.UID, err)
                        return err
                }</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Job &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        task.Job, s.ssn.UID)
                return fmt.Errorf("failed to find job %s", task.Job)
        }</span>

        <span class="cov0" title="0">task.NodeName = hostname
        if node, found := s.ssn.Nodes[hostname]; found </span><span class="cov0" title="0">{
                if err := node.AddTask(task); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to add task &lt;%v/%v&gt; to node &lt;%v&gt; in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, hostname, s.ssn.UID, err)
                        return err
                }</span>
                <span class="cov0" title="0">klog.V(3).Infof("After allocated Task &lt;%v/%v&gt; to Node &lt;%v&gt;: idle &lt;%v&gt;, used &lt;%v&gt;, releasing &lt;%v&gt;",
                        task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing)</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Node &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        hostname, s.ssn.UID)
                return fmt.Errorf("failed to find node %s", hostname)
        }</span>

        // Callbacks
        <span class="cov0" title="0">for _, eh := range s.ssn.eventHandlers </span><span class="cov0" title="0">{
                if eh.AllocateFunc != nil </span><span class="cov0" title="0">{
                        eh.AllocateFunc(&amp;Event{
                                Task: task,
                        })
                }</span>
        }

        // Update status in session
        <span class="cov0" title="0">klog.V(3).Info("Allocating operations ...")
        s.operations = append(s.operations, operation{
                name: Allocate,
                task: task,
        })

        return nil</span>
}

func (s *Statement) allocate(task *api.TaskInfo) error <span class="cov0" title="0">{
        if err := s.ssn.cache.BindVolumes(task, task.PodVolumes); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov0" title="0">if err := s.ssn.cache.Bind(task, task.NodeName); err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        // Update status in session
        <span class="cov0" title="0">if job, found := s.ssn.Jobs[task.Job]; found </span><span class="cov0" title="0">{
                if err := job.UpdateTaskStatus(task, api.Binding); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; status to %v in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, api.Binding, s.ssn.UID, err)
                        return err
                }</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to found Job &lt;%s&gt; in Session &lt;%s&gt; index when binding.",
                        task.Job, s.ssn.UID)
                return fmt.Errorf("failed to find job %s", task.Job)
        }</span>

        <span class="cov0" title="0">metrics.UpdateTaskScheduleDuration(metrics.Duration(task.Pod.CreationTimestamp.Time))
        return nil</span>
}

// unallocate the pod for task
func (s *Statement) unallocate(task *api.TaskInfo) error <span class="cov0" title="0">{
        // Update status in session
        job, found := s.ssn.Jobs[task.Job]
        if found </span><span class="cov0" title="0">{
                if err := job.UpdateTaskStatus(task, api.Pending); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to update task &lt;%v/%v&gt; status to %v in Session &lt;%v&gt;: %v",
                                task.Namespace, task.Name, api.Pending, s.ssn.UID, err)
                }</span>
        } else<span class="cov0" title="0"> {
                klog.Errorf("Failed to find Job &lt;%s&gt; in Session &lt;%s&gt; index when unallocating.",
                        task.Job, s.ssn.UID)
        }</span>

        <span class="cov0" title="0">if node, found := s.ssn.Nodes[task.NodeName]; found </span><span class="cov0" title="0">{
                klog.V(3).Infof("Remove Task &lt;%v&gt; on node &lt;%v&gt;", task.Name, task.NodeName)
                err := node.RemoveTask(task)
                if err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to remove Task &lt;%v&gt; on node &lt;%v&gt;: %s", task.Name, task.NodeName, err.Error())
                }</span>
        }

        <span class="cov0" title="0">for _, eh := range s.ssn.eventHandlers </span><span class="cov0" title="0">{
                if eh.DeallocateFunc != nil </span><span class="cov0" title="0">{
                        eh.DeallocateFunc(&amp;Event{
                                Task: task,
                        })
                }</span>
        }
        <span class="cov0" title="0">task.NodeName = ""

        return nil</span>
}

// Discard operation for evict, pipeline and allocate
func (s *Statement) Discard() <span class="cov0" title="0">{
        klog.V(3).Info("Discarding operations ...")
        for i := len(s.operations) - 1; i &gt;= 0; i-- </span><span class="cov0" title="0">{
                op := s.operations[i]
                op.task.GenerateLastTxContext()
                switch op.name </span>{
                case Evict:<span class="cov0" title="0">
                        err := s.unevict(op.task)
                        if err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("Failed to unevict task: %s", err.Error())
                        }</span>
                case Pipeline:<span class="cov0" title="0">
                        err := s.unpipeline(op.task)
                        if err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("Failed to unpipeline task: %s", err.Error())
                        }</span>
                case Allocate:<span class="cov0" title="0">
                        err := s.unallocate(op.task)
                        if err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("Failed to unallocate task: %s", err.Error())
                        }</span>
                }
        }
}

// Commit operation for evict and pipeline
func (s *Statement) Commit() <span class="cov0" title="0">{
        klog.V(3).Info("Committing operations ...")
        for _, op := range s.operations </span><span class="cov0" title="0">{
                op.task.ClearLastTxContext()
                switch op.name </span>{
                case Evict:<span class="cov0" title="0">
                        err := s.evict(op.task, op.reason)
                        if err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("Failed to evict task: %s", err.Error())
                        }</span>
                case Pipeline:<span class="cov0" title="0">
                        s.pipeline(op.task)</span>
                case Allocate:<span class="cov0" title="0">
                        err := s.allocate(op.task)
                        if err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("Failed to allocate task: for %s", err.Error())
                        }</span>
                }
        }
}
</pre>
		
		<pre class="file" id="file63" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package binpack

import (
        "fmt"
        "strings"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
)

const (
        // PluginName indicates name of volcano scheduler plugin.
        PluginName = "binpack"
)

const (
        // BinpackWeight is the key for providing Binpack Priority Weight in YAML
        BinpackWeight = "binpack.weight"
        // BinpackCPU is the key for weight of cpu
        BinpackCPU = "binpack.cpu"
        // BinpackMemory is the key for weight of memory
        BinpackMemory = "binpack.memory"

        // BinpackResources is the key for additional resource key name
        BinpackResources = "binpack.resources"
        // BinpackResourcesPrefix is the key prefix for additional resource key name
        BinpackResourcesPrefix = BinpackResources + "."

        resourceFmt = "%s[%d]"
)

type priorityWeight struct {
        BinPackingWeight    int
        BinPackingCPU       int
        BinPackingMemory    int
        BinPackingResources map[v1.ResourceName]int
}

func (w *priorityWeight) String() string <span class="cov0" title="0">{
        length := 3
        if extendLength := len(w.BinPackingResources); extendLength == 0 </span><span class="cov0" title="0">{
                length++
        }</span> else<span class="cov0" title="0"> {
                length += extendLength
        }</span>
        <span class="cov0" title="0">msg := make([]string, 0, length)
        msg = append(msg,
                fmt.Sprintf(resourceFmt, BinpackWeight, w.BinPackingWeight),
                fmt.Sprintf(resourceFmt, BinpackCPU, w.BinPackingCPU),
                fmt.Sprintf(resourceFmt, BinpackMemory, w.BinPackingMemory),
        )

        if len(w.BinPackingResources) == 0 </span><span class="cov0" title="0">{
                msg = append(msg, "no extend resources.")
        }</span> else<span class="cov0" title="0"> {
                for name, weight := range w.BinPackingResources </span><span class="cov0" title="0">{
                        msg = append(msg, fmt.Sprintf(resourceFmt, name, weight))
                }</span>
        }
        <span class="cov0" title="0">return strings.Join(msg, ", ")</span>
}

type binpackPlugin struct {
        // Arguments given for the plugin
        weight priorityWeight
}

//New function returns prioritizePlugin object
func New(aruguments framework.Arguments) framework.Plugin <span class="cov8" title="1">{
        weight := calculateWeight(aruguments)
        return &amp;binpackPlugin{weight: weight}
}</span>

func calculateWeight(args framework.Arguments) priorityWeight <span class="cov8" title="1">{
        /*
           User Should give priorityWeight in this format(binpack.weight, binpack.cpu, binpack.memory).
           Support change the weight about cpu, memory and additional resource by arguments.

           actions: "enqueue, reclaim, allocate, backfill, preempt"
           tiers:
           - plugins:
             - name: binpack
               arguments:
                 binpack.weight: 10
                 binpack.cpu: 5
                 binpack.memory: 1
                 binpack.resources: nvidia.com/gpu, example.com/foo
                 binpack.resources.nvidia.com/gpu: 2
                 binpack.resources.example.com/foo: 3
        */
        // Values are initialized to 1.
        weight := priorityWeight{
                BinPackingWeight:    1,
                BinPackingCPU:       1,
                BinPackingMemory:    1,
                BinPackingResources: make(map[v1.ResourceName]int),
        }

        // Checks whether binpack.weight is provided or not, if given, modifies the value in weight struct.
        args.GetInt(&amp;weight.BinPackingWeight, BinpackWeight)
        // Checks whether binpack.cpu is provided or not, if given, modifies the value in weight struct.
        args.GetInt(&amp;weight.BinPackingCPU, BinpackCPU)
        if weight.BinPackingCPU &lt; 0 </span><span class="cov0" title="0">{
                weight.BinPackingCPU = 1
        }</span>
        // Checks whether binpack.memory is provided or not, if given, modifies the value in weight struct.
        <span class="cov8" title="1">args.GetInt(&amp;weight.BinPackingMemory, BinpackMemory)
        if weight.BinPackingMemory &lt; 0 </span><span class="cov0" title="0">{
                weight.BinPackingMemory = 1
        }</span>

        <span class="cov8" title="1">resourcesStr := args[BinpackResources]
        resources := strings.Split(resourcesStr, ",")
        for _, resource := range resources </span><span class="cov8" title="1">{
                resource = strings.TrimSpace(resource)
                if resource == "" </span><span class="cov0" title="0">{
                        continue</span>
                }

                // binpack.resources.[ResourceName]
                <span class="cov8" title="1">resourceKey := BinpackResourcesPrefix + resource
                resourceWeight := 1
                args.GetInt(&amp;resourceWeight, resourceKey)
                if resourceWeight &lt; 0 </span><span class="cov8" title="1">{
                        resourceWeight = 1
                }</span>
                <span class="cov8" title="1">weight.BinPackingResources[v1.ResourceName(resource)] = resourceWeight</span>
        }

        <span class="cov8" title="1">return weight</span>
}

func (bp *binpackPlugin) Name() string <span class="cov8" title="1">{
        return PluginName
}</span>

func (bp *binpackPlugin) OnSessionOpen(ssn *framework.Session) <span class="cov8" title="1">{
        klog.V(4).Infof("Enter binpack plugin ...")
        if klog.V(4) </span><span class="cov0" title="0">{
                defer func() </span><span class="cov0" title="0">{
                        klog.V(4).Infof("Leaving binpack plugin. %s ...", bp.weight.String())
                }</span>()

                <span class="cov0" title="0">notFoundResource := []string{}
                for resource := range bp.weight.BinPackingResources </span><span class="cov0" title="0">{
                        found := false
                        for _, nodeInfo := range ssn.Nodes </span><span class="cov0" title="0">{
                                if nodeInfo.Allocatable.Get(resource) &gt; 0 </span><span class="cov0" title="0">{
                                        found = true
                                        break</span>
                                }
                        }
                        <span class="cov0" title="0">if !found </span><span class="cov0" title="0">{
                                notFoundResource = append(notFoundResource, string(resource))
                        }</span>
                }
                <span class="cov0" title="0">klog.V(4).Infof("resources [%s] record in weight but not found on any node", strings.Join(notFoundResource, ", "))</span>
        }

        <span class="cov8" title="1">nodeOrderFn := func(task *api.TaskInfo, node *api.NodeInfo) (float64, error) </span><span class="cov8" title="1">{
                binPackingScore := BinPackingScore(task, node, bp.weight)

                klog.V(4).Infof("Binpack score for Task %s/%s on node %s is: %v", task.Namespace, task.Name, node.Name, binPackingScore)
                return binPackingScore, nil
        }</span>
        <span class="cov8" title="1">if bp.weight.BinPackingWeight != 0 </span><span class="cov8" title="1">{
                ssn.AddNodeOrderFn(bp.Name(), nodeOrderFn)
        }</span> else<span class="cov0" title="0"> {
                klog.Infof("binpack weight is zero, skip node order function")
        }</span>
}

func (bp *binpackPlugin) OnSessionClose(ssn *framework.Session) {<span class="cov8" title="1">
}</span>

// BinPackingScore use the best fit polices during scheduling.
// Goals:
// - Schedule Jobs using BestFit Policy using Resource Bin Packing Priority Function
// - Reduce Fragmentation of scarce resources on the Cluster
func BinPackingScore(task *api.TaskInfo, node *api.NodeInfo, weight priorityWeight) float64 <span class="cov8" title="1">{
        score := 0.0
        weightSum := 0
        requested := task.Resreq
        allocatable := node.Allocatable
        used := node.Used

        for _, resource := range requested.ResourceNames() </span><span class="cov8" title="1">{
                request := requested.Get(resource)
                if request == 0 </span><span class="cov0" title="0">{
                        continue</span>
                }
                <span class="cov8" title="1">allocate := allocatable.Get(resource)
                nodeUsed := used.Get(resource)

                resourceWeight := 0
                found := false
                switch resource </span>{
                case v1.ResourceCPU:<span class="cov8" title="1">
                        resourceWeight = weight.BinPackingCPU
                        found = true</span>
                case v1.ResourceMemory:<span class="cov8" title="1">
                        resourceWeight = weight.BinPackingMemory
                        found = true</span>
                default:<span class="cov8" title="1">
                        resourceWeight, found = weight.BinPackingResources[resource]</span>
                }
                <span class="cov8" title="1">if !found </span><span class="cov8" title="1">{
                        continue</span>
                }

                <span class="cov8" title="1">resourceScore := ResourceBinPackingScore(request, allocate, nodeUsed, resourceWeight)
                klog.V(5).Infof("task %s/%s on node %s resource %s, need %f, used %f, allocatable %f, weight %d, score %f", task.Namespace, task.Name, node.Name, resource, request, nodeUsed, allocate, resourceWeight, resourceScore)

                score += resourceScore
                weightSum += resourceWeight</span>
        }

        // mapping the result from [0, weightSum] to [0, 10(MaxPriority)]
        <span class="cov8" title="1">if weightSum &gt; 0 </span><span class="cov8" title="1">{
                score /= float64(weightSum)
        }</span>
        <span class="cov8" title="1">score *= float64(v1alpha1.MaxNodeScore * int64(weight.BinPackingWeight))

        return score</span>
}

// ResourceBinPackingScore calculate the binpack score for resource with provided info
func ResourceBinPackingScore(requested, capacity, used float64, weight int) float64 <span class="cov8" title="1">{
        if capacity == 0 || weight == 0 </span><span class="cov8" title="1">{
                return 0
        }</span>

        <span class="cov8" title="1">usedFinally := requested + used
        if usedFinally &gt; capacity </span><span class="cov8" title="1">{
                return 0
        }</span>

        <span class="cov8" title="1">score := usedFinally * float64(weight) / capacity
        return score</span>
}
</pre>
		
		<pre class="file" id="file64" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package drf

import (
        "fmt"
        "math"
        "strconv"
        "strings"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/api/helpers"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/metrics"
        "volcano.sh/volcano/pkg/scheduler/plugins/util"
)

// PluginName indicates name of volcano scheduler plugin.
const PluginName = "drf"

var shareDelta = 0.000001

// hierarchicalNode represents the node hierarchy
// and the corresponding weight and drf attribute
type hierarchicalNode struct {
        parent *hierarchicalNode
        attr   *drfAttr
        // If the node is a leaf node,
        // request represents the request of the job.
        request   *api.Resource
        weight    float64
        saturated bool
        hierarchy string
        children  map[string]*hierarchicalNode
}

func (node *hierarchicalNode) Clone(parent *hierarchicalNode) *hierarchicalNode <span class="cov0" title="0">{
        newNode := &amp;hierarchicalNode{
                parent: parent,
                attr: &amp;drfAttr{
                        share:            node.attr.share,
                        dominantResource: node.attr.dominantResource,
                        allocated:        node.attr.allocated.Clone(),
                },
                request:   node.request.Clone(),
                weight:    node.weight,
                saturated: node.saturated,
                hierarchy: node.hierarchy,
                children:  nil,
        }
        if node.children != nil </span><span class="cov0" title="0">{
                newNode.children = map[string]*hierarchicalNode{}
                for _, child := range node.children </span><span class="cov0" title="0">{
                        newNode.children[child.hierarchy] = child.Clone(newNode)
                }</span>
        }
        <span class="cov0" title="0">return newNode</span>
}

// resourceSaturated returns true if any resource of the job is saturated or the job demands fully allocated resource
func resourceSaturated(allocated *api.Resource,
        jobRequest *api.Resource, demandingResources map[v1.ResourceName]bool) bool <span class="cov8" title="1">{
        for _, rn := range allocated.ResourceNames() </span><span class="cov8" title="1">{
                if allocated.Get(rn) != 0 &amp;&amp; jobRequest.Get(rn) != 0 &amp;&amp;
                        allocated.Get(rn) &gt;= jobRequest.Get(rn) </span><span class="cov0" title="0">{
                        return true
                }</span>
                <span class="cov8" title="1">if !demandingResources[rn] &amp;&amp; jobRequest.Get(rn) != 0 </span><span class="cov8" title="1">{
                        return true
                }</span>
        }
        <span class="cov8" title="1">return false</span>
}

type drfAttr struct {
        share            float64
        dominantResource string
        allocated        *api.Resource
}

func (attr *drfAttr) String() string <span class="cov8" title="1">{
        return fmt.Sprintf("dominant resource &lt;%s&gt;, dominant share %f, allocated %s",
                attr.dominantResource, attr.share, attr.allocated)
}</span>

type drfPlugin struct {
        totalResource  *api.Resource
        totalAllocated *api.Resource

        // Key is Job ID
        jobAttrs map[api.JobID]*drfAttr

        // map[namespaceName]-&gt;attr
        namespaceOpts map[string]*drfAttr

        // hierarchical tree root
        hierarchicalRoot *hierarchicalNode

        // Arguments given for the plugin
        pluginArguments framework.Arguments
}

// New return drf plugin
func New(arguments framework.Arguments) framework.Plugin <span class="cov8" title="1">{
        return &amp;drfPlugin{
                totalResource:  api.EmptyResource(),
                totalAllocated: api.EmptyResource(),
                jobAttrs:       map[api.JobID]*drfAttr{},
                namespaceOpts:  map[string]*drfAttr{},
                hierarchicalRoot: &amp;hierarchicalNode{
                        attr:      &amp;drfAttr{allocated: api.EmptyResource()},
                        request:   api.EmptyResource(),
                        hierarchy: "root",
                        weight:    1,
                        children:  map[string]*hierarchicalNode{},
                },
                pluginArguments: arguments,
        }
}</span>

func (drf *drfPlugin) Name() string <span class="cov8" title="1">{
        return PluginName
}</span>

// HierarchyEnabled returns if hierarchy is enabled
func (drf *drfPlugin) HierarchyEnabled(ssn *framework.Session) bool <span class="cov8" title="1">{
        for _, tier := range ssn.Tiers </span><span class="cov8" title="1">{
                for _, plugin := range tier.Plugins </span><span class="cov8" title="1">{
                        if plugin.Name != PluginName </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov8" title="1">return plugin.EnabledHierarchy != nil &amp;&amp; *plugin.EnabledHierarchy</span>
                }
        }
        <span class="cov0" title="0">return false</span>
}

// NamespaceOrderEnabled returns the NamespaceOrder for this plugin is enabled in this session or not
func (drf *drfPlugin) NamespaceOrderEnabled(ssn *framework.Session) bool <span class="cov8" title="1">{
        for _, tier := range ssn.Tiers </span><span class="cov8" title="1">{
                for _, plugin := range tier.Plugins </span><span class="cov8" title="1">{
                        if plugin.Name != PluginName </span><span class="cov0" title="0">{
                                continue</span>
                        }
                        <span class="cov8" title="1">return plugin.EnabledNamespaceOrder != nil &amp;&amp; *plugin.EnabledNamespaceOrder</span>
                }
        }
        <span class="cov0" title="0">return false</span>
}

func (drf *drfPlugin) compareQueues(root *hierarchicalNode, lqueue *api.QueueInfo, rqueue *api.QueueInfo) float64 <span class="cov8" title="1">{
        lnode := root
        lpaths := strings.Split(lqueue.Hierarchy, "/")
        rnode := root
        rpaths := strings.Split(rqueue.Hierarchy, "/")
        depth := 0
        if len(lpaths) &lt; len(rpaths) </span><span class="cov8" title="1">{
                depth = len(lpaths)
        }</span> else<span class="cov8" title="1"> {
                depth = len(rpaths)
        }</span>
        <span class="cov8" title="1">for i := 0; i &lt; depth; i++ </span><span class="cov8" title="1">{
                // Saturated nodes have minumun prioirty,
                // so that demanding nodes will be poped first.
                if !lnode.saturated &amp;&amp; rnode.saturated </span><span class="cov0" title="0">{
                        return -1
                }</span>
                <span class="cov8" title="1">if lnode.saturated &amp;&amp; !rnode.saturated </span><span class="cov0" title="0">{
                        return 1
                }</span>
                <span class="cov8" title="1">if lnode.attr.share/lnode.weight == rnode.attr.share/rnode.weight </span><span class="cov8" title="1">{
                        if i &lt; depth-1 </span><span class="cov8" title="1">{
                                lnode = lnode.children[lpaths[i+1]]
                                rnode = rnode.children[rpaths[i+1]]
                        }</span>
                } else<span class="cov8" title="1"> {
                        return lnode.attr.share/lnode.weight - rnode.attr.share/rnode.weight
                }</span>
        }
        <span class="cov8" title="1">return 0</span>
}

func (drf *drfPlugin) OnSessionOpen(ssn *framework.Session) <span class="cov8" title="1">{
        // Prepare scheduling data for this session.
        drf.totalResource.Add(ssn.TotalResource)

        klog.V(4).Infof("Total Allocatable %s", drf.totalResource)

        namespaceOrderEnabled := drf.NamespaceOrderEnabled(ssn)
        hierarchyEnabled := drf.HierarchyEnabled(ssn)

        for _, job := range ssn.Jobs </span><span class="cov8" title="1">{
                attr := &amp;drfAttr{
                        allocated: api.EmptyResource(),
                }

                for status, tasks := range job.TaskStatusIndex </span><span class="cov8" title="1">{
                        if api.AllocatedStatus(status) </span><span class="cov0" title="0">{
                                for _, t := range tasks </span><span class="cov0" title="0">{
                                        attr.allocated.Add(t.Resreq)
                                }</span>
                        }
                }

                // Calculate the init share of Job
                <span class="cov8" title="1">drf.updateJobShare(job.Namespace, job.Name, attr)

                drf.jobAttrs[job.UID] = attr

                if namespaceOrderEnabled </span><span class="cov0" title="0">{
                        nsOpts, found := drf.namespaceOpts[job.Namespace]
                        if !found </span><span class="cov0" title="0">{
                                nsOpts = &amp;drfAttr{
                                        allocated: api.EmptyResource(),
                                }
                                drf.namespaceOpts[job.Namespace] = nsOpts
                        }</span>
                        // all task in job should have the same namespace with job
                        <span class="cov0" title="0">nsOpts.allocated.Add(attr.allocated)
                        drf.updateNamespaceShare(job.Namespace, nsOpts)</span>
                }
                <span class="cov8" title="1">if hierarchyEnabled </span><span class="cov8" title="1">{
                        queue := ssn.Queues[job.Queue]
                        drf.totalAllocated.Add(attr.allocated)
                        drf.UpdateHierarchicalShare(drf.hierarchicalRoot, drf.totalAllocated, job, attr, queue.Hierarchy, queue.Weights)
                }</span>
        }

        <span class="cov8" title="1">preemptableFn := func(preemptor *api.TaskInfo, preemptees []*api.TaskInfo) ([]*api.TaskInfo, int) </span><span class="cov0" title="0">{
                var victims []*api.TaskInfo

                addVictim := func(candidate *api.TaskInfo) </span><span class="cov0" title="0">{
                        victims = append(victims, candidate)
                }</span>

                <span class="cov0" title="0">if namespaceOrderEnabled </span><span class="cov0" title="0">{
                        // apply the namespace share policy on preemptee firstly

                        lWeight := ssn.NamespaceInfo[api.NamespaceName(preemptor.Namespace)].GetWeight()
                        lNsAtt := drf.namespaceOpts[preemptor.Namespace]
                        lNsAlloc := lNsAtt.allocated.Clone().Add(preemptor.Resreq)
                        _, lNsShare := drf.calculateShare(lNsAlloc, drf.totalResource)
                        lNsShareWeighted := lNsShare / float64(lWeight)

                        namespaceAllocation := map[string]*api.Resource{}

                        // undecidedPreemptees means this policy could not judge preemptee is preemptable or not
                        // and left it to next policy
                        undecidedPreemptees := []*api.TaskInfo{}

                        for _, preemptee := range preemptees </span><span class="cov0" title="0">{
                                if preemptor.Namespace == preemptee.Namespace </span><span class="cov0" title="0">{
                                        // policy is disabled when they are in the same namespace
                                        undecidedPreemptees = append(undecidedPreemptees, preemptee)
                                        continue</span>
                                }

                                // compute the preemptee namespace weighted share after preemption
                                <span class="cov0" title="0">nsAllocation, found := namespaceAllocation[preemptee.Namespace]
                                if !found </span><span class="cov0" title="0">{
                                        rNsAtt := drf.namespaceOpts[preemptee.Namespace]
                                        nsAllocation = rNsAtt.allocated.Clone()
                                        namespaceAllocation[preemptee.Namespace] = nsAllocation
                                }</span>
                                <span class="cov0" title="0">rWeight := ssn.NamespaceInfo[api.NamespaceName(preemptee.Namespace)].GetWeight()
                                rNsAlloc := nsAllocation.Sub(preemptee.Resreq)
                                _, rNsShare := drf.calculateShare(rNsAlloc, drf.totalResource)
                                rNsShareWeighted := rNsShare / float64(rWeight)

                                // to avoid ping pong actions, the preemptee namespace should
                                // have the higher weighted share after preemption.
                                if lNsShareWeighted &lt; rNsShareWeighted </span><span class="cov0" title="0">{
                                        addVictim(preemptee)
                                        continue</span>
                                }
                                <span class="cov0" title="0">if lNsShareWeighted-rNsShareWeighted &gt; shareDelta </span><span class="cov0" title="0">{
                                        continue</span>
                                }

                                // equal namespace order leads to judgement of jobOrder
                                <span class="cov0" title="0">undecidedPreemptees = append(undecidedPreemptees, preemptee)</span>
                        }

                        <span class="cov0" title="0">preemptees = undecidedPreemptees</span>
                }

                <span class="cov0" title="0">latt := drf.jobAttrs[preemptor.Job]
                lalloc := latt.allocated.Clone().Add(preemptor.Resreq)
                _, ls := drf.calculateShare(lalloc, drf.totalResource)

                allocations := map[api.JobID]*api.Resource{}

                for _, preemptee := range preemptees </span><span class="cov0" title="0">{
                        if _, found := allocations[preemptee.Job]; !found </span><span class="cov0" title="0">{
                                ratt := drf.jobAttrs[preemptee.Job]
                                allocations[preemptee.Job] = ratt.allocated.Clone()
                        }</span>
                        <span class="cov0" title="0">ralloc := allocations[preemptee.Job].Sub(preemptee.Resreq)
                        _, rs := drf.calculateShare(ralloc, drf.totalResource)

                        if ls &lt; rs || math.Abs(ls-rs) &lt;= shareDelta </span><span class="cov0" title="0">{
                                addVictim(preemptee)
                        }</span>
                }

                <span class="cov0" title="0">klog.V(4).Infof("Victims from DRF plugins are %+v", victims)

                return victims, util.Permit</span>
        }

        <span class="cov8" title="1">ssn.AddPreemptableFn(drf.Name(), preemptableFn)

        if hierarchyEnabled </span><span class="cov8" title="1">{
                queueOrderFn := func(l interface{}, r interface{}) int </span><span class="cov8" title="1">{
                        lv := l.(*api.QueueInfo)
                        rv := r.(*api.QueueInfo)
                        ret := drf.compareQueues(drf.hierarchicalRoot, lv, rv)
                        if ret &lt; 0 </span><span class="cov8" title="1">{
                                return -1
                        }</span>
                        <span class="cov8" title="1">if ret &gt; 0 </span><span class="cov8" title="1">{
                                return 1
                        }</span>
                        <span class="cov8" title="1">return 0</span>
                }
                <span class="cov8" title="1">ssn.AddQueueOrderFn(drf.Name(), queueOrderFn)

                reclaimFn := func(reclaimer *api.TaskInfo, reclaimees []*api.TaskInfo) ([]*api.TaskInfo, int) </span><span class="cov0" title="0">{
                        var victims []*api.TaskInfo
                        // clone hdrf tree
                        totalAllocated := drf.totalAllocated.Clone()
                        root := drf.hierarchicalRoot.Clone(nil)

                        //  update reclaimer hdrf
                        ljob := ssn.Jobs[reclaimer.Job]
                        lqueue := ssn.Queues[ljob.Queue]
                        ljob = ljob.Clone()
                        attr := drf.jobAttrs[ljob.UID]
                        lattr := &amp;drfAttr{
                                allocated: attr.allocated.Clone(),
                        }
                        lattr.allocated.Add(reclaimer.Resreq)
                        totalAllocated.Add(reclaimer.Resreq)
                        drf.updateShare(lattr)
                        drf.UpdateHierarchicalShare(root, totalAllocated, ljob, lattr, lqueue.Hierarchy, lqueue.Weights)

                        for _, preemptee := range reclaimees </span><span class="cov0" title="0">{
                                rjob := ssn.Jobs[preemptee.Job]
                                rqueue := ssn.Queues[rjob.Queue]

                                // update hdrf of reclaimee job
                                totalAllocated.Sub(preemptee.Resreq)
                                rjob = rjob.Clone()
                                attr := drf.jobAttrs[rjob.UID]
                                rattr := &amp;drfAttr{
                                        allocated: attr.allocated.Clone(),
                                }
                                rattr.allocated.Sub(preemptee.Resreq)
                                drf.updateShare(rattr)
                                drf.UpdateHierarchicalShare(root, totalAllocated, rjob, rattr, rqueue.Hierarchy, rqueue.Weights)

                                // compare hdrf of queues
                                ret := drf.compareQueues(root, lqueue, rqueue)

                                // resume hdrf of reclaimee job
                                totalAllocated.Add(preemptee.Resreq)
                                rattr.allocated.Add(preemptee.Resreq)
                                drf.updateShare(rattr)
                                drf.UpdateHierarchicalShare(root, totalAllocated, rjob, rattr, rqueue.Hierarchy, rqueue.Weights)

                                if ret &lt; 0 </span><span class="cov0" title="0">{
                                        victims = append(victims, preemptee)
                                }</span>

                                <span class="cov0" title="0">if ret &gt; shareDelta </span><span class="cov0" title="0">{
                                        continue</span>
                                }
                        }

                        <span class="cov0" title="0">klog.V(4).Infof("Victims from HDRF plugins are %+v", victims)

                        return victims, util.Permit</span>
                }
                <span class="cov8" title="1">ssn.AddReclaimableFn(drf.Name(), reclaimFn)</span>
        }

        <span class="cov8" title="1">jobOrderFn := func(l interface{}, r interface{}) int </span><span class="cov0" title="0">{
                lv := l.(*api.JobInfo)
                rv := r.(*api.JobInfo)

                klog.V(4).Infof("DRF JobOrderFn: &lt;%v/%v&gt; share state: %v, &lt;%v/%v&gt; share state: %v",
                        lv.Namespace, lv.Name, drf.jobAttrs[lv.UID].share, rv.Namespace, rv.Name, drf.jobAttrs[rv.UID].share)

                if drf.jobAttrs[lv.UID].share == drf.jobAttrs[rv.UID].share </span><span class="cov0" title="0">{
                        return 0
                }</span>

                <span class="cov0" title="0">if drf.jobAttrs[lv.UID].share &lt; drf.jobAttrs[rv.UID].share </span><span class="cov0" title="0">{
                        return -1
                }</span>

                <span class="cov0" title="0">return 1</span>
        }

        <span class="cov8" title="1">ssn.AddJobOrderFn(drf.Name(), jobOrderFn)

        namespaceOrderFn := func(l interface{}, r interface{}) int </span><span class="cov0" title="0">{
                lv := l.(api.NamespaceName)
                rv := r.(api.NamespaceName)

                lOpt := drf.namespaceOpts[string(lv)]
                rOpt := drf.namespaceOpts[string(rv)]

                lWeight := ssn.NamespaceInfo[lv].GetWeight()
                rWeight := ssn.NamespaceInfo[rv].GetWeight()

                klog.V(4).Infof("DRF NamespaceOrderFn: &lt;%v&gt; share state: %f, weight %v, &lt;%v&gt; share state: %f, weight %v",
                        lv, lOpt.share, lWeight, rv, rOpt.share, rWeight)

                lWeightedShare := lOpt.share / float64(lWeight)
                rWeightedShare := rOpt.share / float64(rWeight)

                metrics.UpdateNamespaceWeight(string(lv), lWeight)
                metrics.UpdateNamespaceWeight(string(rv), rWeight)
                metrics.UpdateNamespaceWeightedShare(string(lv), lWeightedShare)
                metrics.UpdateNamespaceWeightedShare(string(rv), rWeightedShare)

                if lWeightedShare == rWeightedShare </span><span class="cov0" title="0">{
                        return 0
                }</span>

                <span class="cov0" title="0">if lWeightedShare &lt; rWeightedShare </span><span class="cov0" title="0">{
                        return -1
                }</span>

                <span class="cov0" title="0">return 1</span>
        }

        <span class="cov8" title="1">if namespaceOrderEnabled </span><span class="cov0" title="0">{
                ssn.AddNamespaceOrderFn(drf.Name(), namespaceOrderFn)
        }</span>

        // Register event handlers.
        <span class="cov8" title="1">ssn.AddEventHandler(&amp;framework.EventHandler{
                AllocateFunc: func(event *framework.Event) </span><span class="cov8" title="1">{
                        attr := drf.jobAttrs[event.Task.Job]
                        attr.allocated.Add(event.Task.Resreq)

                        job := ssn.Jobs[event.Task.Job]
                        drf.updateJobShare(job.Namespace, job.Name, attr)

                        nsShare := -1.0
                        if namespaceOrderEnabled </span><span class="cov0" title="0">{
                                nsOpt := drf.namespaceOpts[event.Task.Namespace]
                                nsOpt.allocated.Add(event.Task.Resreq)

                                drf.updateNamespaceShare(event.Task.Namespace, nsOpt)
                                nsShare = nsOpt.share
                        }</span>
                        <span class="cov8" title="1">if hierarchyEnabled </span><span class="cov8" title="1">{
                                queue := ssn.Queues[job.Queue]

                                drf.totalAllocated.Add(event.Task.Resreq)
                                drf.UpdateHierarchicalShare(drf.hierarchicalRoot, drf.totalAllocated, job, attr, queue.Hierarchy, queue.Weights)
                        }</span>

                        <span class="cov8" title="1">klog.V(4).Infof("DRF AllocateFunc: task &lt;%v/%v&gt;, resreq &lt;%v&gt;,  share &lt;%v&gt;, namespace share &lt;%v&gt;",
                                event.Task.Namespace, event.Task.Name, event.Task.Resreq, attr.share, nsShare)</span>
                },
                DeallocateFunc: func(event *framework.Event) <span class="cov0" title="0">{
                        attr := drf.jobAttrs[event.Task.Job]
                        attr.allocated.Sub(event.Task.Resreq)

                        job := ssn.Jobs[event.Task.Job]
                        drf.updateJobShare(job.Namespace, job.Name, attr)

                        nsShare := -1.0
                        if namespaceOrderEnabled </span><span class="cov0" title="0">{
                                nsOpt := drf.namespaceOpts[event.Task.Namespace]
                                nsOpt.allocated.Sub(event.Task.Resreq)

                                drf.updateNamespaceShare(event.Task.Namespace, nsOpt)
                                nsShare = nsOpt.share
                        }</span>

                        <span class="cov0" title="0">if hierarchyEnabled </span><span class="cov0" title="0">{
                                queue := ssn.Queues[job.Queue]
                                drf.totalAllocated.Sub(event.Task.Resreq)
                                drf.UpdateHierarchicalShare(drf.hierarchicalRoot, drf.totalAllocated, job, attr, queue.Hierarchy, queue.Weights)
                        }</span>

                        <span class="cov0" title="0">klog.V(4).Infof("DRF EvictFunc: task &lt;%v/%v&gt;, resreq &lt;%v&gt;,  share &lt;%v&gt;, namespace share &lt;%v&gt;",
                                event.Task.Namespace, event.Task.Name, event.Task.Resreq, attr.share, nsShare)</span>
                },
        })
}

func (drf *drfPlugin) updateNamespaceShare(namespaceName string, attr *drfAttr) <span class="cov0" title="0">{
        drf.updateShare(attr)
        metrics.UpdateNamespaceShare(namespaceName, attr.share)
}</span>

// build hierarchy if the node does not exist
func (drf *drfPlugin) buildHierarchy(root *hierarchicalNode, job *api.JobInfo, attr *drfAttr,
        hierarchy, hierarchicalWeights string) <span class="cov8" title="1">{
        inode := root
        paths := strings.Split(hierarchy, "/")
        weights := strings.Split(hierarchicalWeights, "/")

        for i := 1; i &lt; len(paths); i++ </span><span class="cov8" title="1">{
                if child, ok := inode.children[paths[i]]; ok </span><span class="cov8" title="1">{
                        inode = child
                }</span> else<span class="cov8" title="1"> {
                        fweight, _ := strconv.ParseFloat(weights[i], 64)
                        if fweight &lt; 1 </span><span class="cov0" title="0">{
                                fweight = 1
                        }</span>
                        <span class="cov8" title="1">child = &amp;hierarchicalNode{
                                weight:    fweight,
                                hierarchy: paths[i],
                                request:   api.EmptyResource(),
                                attr: &amp;drfAttr{
                                        allocated: api.EmptyResource(),
                                },
                                children: make(map[string]*hierarchicalNode),
                        }
                        klog.V(4).Infof("Node %s added to %s, weight %f",
                                child.hierarchy, inode.hierarchy, fweight)
                        inode.children[paths[i]] = child
                        child.parent = inode
                        inode = child</span>
                }
        }

        <span class="cov8" title="1">child := &amp;hierarchicalNode{
                weight:    1,
                attr:      attr,
                hierarchy: string(job.UID),
                request:   job.TotalRequest.Clone(),
                children:  nil,
        }
        inode.children[string(job.UID)] = child
        // update drf attribute bottom up
        klog.V(4).Infof("Job &lt;%s/%s&gt; added to %s, weights %s, attr %v, total request: %s",
                job.Namespace, job.Name, inode.hierarchy, hierarchicalWeights, child.attr, job.TotalRequest)</span>
}

// updateNamespaceShare updates the node attribute recursively
func (drf *drfPlugin) updateHierarchicalShare(node *hierarchicalNode,
        demandingResources map[v1.ResourceName]bool) <span class="cov8" title="1">{
        if node.children == nil </span><span class="cov8" title="1">{
                node.saturated = resourceSaturated(node.attr.allocated,
                        node.request, demandingResources)
                klog.V(4).Infof("Update hierarchical node %s, share %f, dominant %s, resource %v, saturated: %t",
                        node.hierarchy, node.attr.share, node.attr.dominantResource, node.attr.allocated, node.saturated)
        }</span> else<span class="cov8" title="1"> {
                var mdr float64 = 1
                // get minimun dominant resource share
                for _, child := range node.children </span><span class="cov8" title="1">{
                        drf.updateHierarchicalShare(child, demandingResources)
                        // skip empty child and saturated child
                        if child.attr.share != 0 &amp;&amp; !child.saturated </span><span class="cov8" title="1">{
                                _, resShare := drf.calculateShare(child.attr.allocated, drf.totalResource)
                                if resShare &lt; mdr </span><span class="cov8" title="1">{
                                        mdr = resShare
                                }</span>
                        }
                }

                <span class="cov8" title="1">node.attr.allocated = api.EmptyResource()
                saturated := true
                for _, child := range node.children </span><span class="cov8" title="1">{
                        if !child.saturated </span><span class="cov8" title="1">{
                                saturated = false
                        }</span>
                        // only consider non-empty children
                        <span class="cov8" title="1">if child.attr.share != 0 </span><span class="cov8" title="1">{
                                // saturated child is not scaled
                                if child.saturated </span><span class="cov8" title="1">{
                                        t := child.attr.allocated
                                        node.attr.allocated.Add(t)
                                }</span> else<span class="cov8" title="1"> {
                                        t := child.attr.allocated.Clone().Multi(mdr / child.attr.share)
                                        node.attr.allocated.Add(t)
                                }</span>
                        }
                }
                <span class="cov8" title="1">node.attr.dominantResource, node.attr.share = drf.calculateShare(
                        node.attr.allocated, drf.totalResource)
                node.saturated = saturated
                klog.V(4).Infof("Update hierarchical node %s, share %f, dominant resource %s, resource %v, saturated: %t",
                        node.hierarchy, node.attr.share, node.attr.dominantResource, node.attr.allocated, node.saturated)</span>
        }
}

func (drf *drfPlugin) UpdateHierarchicalShare(root *hierarchicalNode, totalAllocated *api.Resource, job *api.JobInfo, attr *drfAttr, hierarchy, hierarchicalWeights string) <span class="cov8" title="1">{
        // filter out demanding resources
        demandingResources := map[v1.ResourceName]bool{}
        for _, rn := range drf.totalResource.ResourceNames() </span><span class="cov8" title="1">{
                if totalAllocated.Get(rn) &lt; drf.totalResource.Get(rn) </span><span class="cov8" title="1">{
                        demandingResources[rn] = true
                }</span>
        }
        <span class="cov8" title="1">drf.buildHierarchy(root, job, attr, hierarchy, hierarchicalWeights)
        drf.updateHierarchicalShare(root, demandingResources)</span>
}

func (drf *drfPlugin) updateJobShare(jobNs, jobName string, attr *drfAttr) <span class="cov8" title="1">{
        drf.updateShare(attr)
        metrics.UpdateJobShare(jobNs, jobName, attr.share)
}</span>

func (drf *drfPlugin) updateShare(attr *drfAttr) <span class="cov8" title="1">{
        attr.dominantResource, attr.share = drf.calculateShare(attr.allocated, drf.totalResource)
}</span>

func (drf *drfPlugin) calculateShare(allocated, totalResource *api.Resource) (string, float64) <span class="cov8" title="1">{
        res := float64(0)
        dominantResource := ""
        for _, rn := range totalResource.ResourceNames() </span><span class="cov8" title="1">{
                share := helpers.Share(allocated.Get(rn), totalResource.Get(rn))
                if share &gt; res </span><span class="cov8" title="1">{
                        res = share
                        dominantResource = string(rn)
                }</span>
        }

        <span class="cov8" title="1">return dominantResource, res</span>
}

func (drf *drfPlugin) OnSessionClose(session *framework.Session) <span class="cov8" title="1">{
        // Clean schedule data.
        drf.totalResource = api.EmptyResource()
        drf.totalAllocated = api.EmptyResource()
        drf.jobAttrs = map[api.JobID]*drfAttr{}
}</span>
</pre>
		
		<pre class="file" id="file65" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import (
        v1 "k8s.io/api/core/v1"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
        "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        nodeinfov1alpha1 "volcano.sh/apis/pkg/apis/nodeinfo/v1alpha1"
        "volcano.sh/volcano/pkg/scheduler/api"
)

// TopologyHint is a struct containing the NUMANodeAffinity for a Container
type TopologyHint struct {
        NUMANodeAffinity bitmask.BitMask
        // Preferred is set to true when the NUMANodeAffinity encodes a preferred
        // allocation for the Container. It is set to false otherwise.
        Preferred bool
}

// Policy is an interface for topology manager policy
type Policy interface {
        // Predicate Get the best hit.
        Predicate(providersHints []map[string][]TopologyHint) (TopologyHint, bool)
}

// HintProvider is an interface for components that want to collaborate to
// achieve globally optimal concrete resource alignment with respect to
// NUMA locality.
type HintProvider interface {
        // Name returns provider name used for register and logging.
        Name() string
        // GetTopologyHints returns hints if this hint provider has a preference,
        GetTopologyHints(container *v1.Container, topoInfo *api.NumatopoInfo, resNumaSets api.ResNumaSets) map[string][]TopologyHint
        Allocate(container *v1.Container, bestHit *TopologyHint, topoInfo *api.NumatopoInfo, resNumaSets api.ResNumaSets) map[string]cpuset.CPUSet
}

// GetPolicy return the interface matched the input task topology config
func GetPolicy(node *api.NodeInfo, numaNodes []int) Policy <span class="cov0" title="0">{
        switch batch.NumaPolicy(node.NumaSchedulerInfo.Policies[nodeinfov1alpha1.TopologyManagerPolicy]) </span>{
        case batch.None:<span class="cov0" title="0">
                return NewPolicyNone(numaNodes)</span>
        case batch.BestEffort:<span class="cov0" title="0">
                return NewPolicyBestEffort(numaNodes)</span>
        case batch.Restricted:<span class="cov0" title="0">
                return NewPolicyRestricted(numaNodes)</span>
        case batch.SingleNumaNode:<span class="cov0" title="0">
                return NewPolicySingleNumaNode(numaNodes)</span>
        }

        <span class="cov0" title="0">return &amp;policyNone{}</span>
}

// AccumulateProvidersHints return all TopologyHint collection from different providers
func AccumulateProvidersHints(container *v1.Container,
        topoInfo *api.NumatopoInfo, resNumaSets api.ResNumaSets,
        hintProviders []HintProvider) (providersHints []map[string][]TopologyHint) <span class="cov0" title="0">{
        for _, provider := range hintProviders </span><span class="cov0" title="0">{
                hints := provider.GetTopologyHints(container, topoInfo, resNumaSets)
                providersHints = append(providersHints, hints)
        }</span>

        <span class="cov0" title="0">return providersHints</span>
}

// Allocate return all resource assignment collection from different providers
func Allocate(container *v1.Container, bestHit *TopologyHint,
        topoInfo *api.NumatopoInfo, resNumaSets api.ResNumaSets, hintProviders []HintProvider) map[string]cpuset.CPUSet <span class="cov0" title="0">{
        allResAlloc := make(map[string]cpuset.CPUSet)
        for _, provider := range hintProviders </span><span class="cov0" title="0">{
                resAlloc := provider.Allocate(container, bestHit, topoInfo, resNumaSets)
                for resName, assign := range resAlloc </span><span class="cov0" title="0">{
                        allResAlloc[resName] = assign
                }</span>
        }

        <span class="cov0" title="0">return allResAlloc</span>
}
</pre>
		
		<pre class="file" id="file66" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import (
        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
)

func filterProvidersHints(providersHints []map[string][]TopologyHint) [][]TopologyHint <span class="cov8" title="1">{
        var allProviderHints [][]TopologyHint
        for _, hints := range providersHints </span><span class="cov8" title="1">{
                // If hints is nil, insert a single, preferred any-numa hint into allProviderHints.
                if len(hints) == 0 </span><span class="cov0" title="0">{
                        klog.Infof("[numatopo] Hint Provider has no preference for NUMA affinity with any resource")
                        allProviderHints = append(allProviderHints, []TopologyHint{{nil, true}})
                        continue</span>
                }

                // Otherwise, accumulate the hints for each resource type into allProviderHints.
                <span class="cov8" title="1">for resource := range hints </span><span class="cov8" title="1">{
                        if hints[resource] == nil </span><span class="cov0" title="0">{
                                klog.Infof("[numatopo] Hint Provider has no preference for NUMA affinity with resource '%s'", resource)
                                allProviderHints = append(allProviderHints, []TopologyHint{{nil, true}})
                                continue</span>
                        }

                        <span class="cov8" title="1">if len(hints[resource]) == 0 </span><span class="cov0" title="0">{
                                klog.Infof("[numatopo] Hint Provider has no possible NUMA affinities for resource '%s'", resource)
                                allProviderHints = append(allProviderHints, []TopologyHint{{nil, false}})
                                continue</span>
                        }

                        <span class="cov8" title="1">allProviderHints = append(allProviderHints, hints[resource])</span>
                }
        }
        <span class="cov8" title="1">return allProviderHints</span>
}

func mergeFilteredHints(numaNodes []int, filteredHints [][]TopologyHint) TopologyHint <span class="cov8" title="1">{
        // Set the default affinity as an any-numa affinity containing the list
        // of NUMA Nodes available on this machine.
        defaultAffinity, _ := bitmask.NewBitMask(numaNodes...)

        // Set the bestHint to return from this function as {nil false}.
        // This will only be returned if no better hint can be found when
        // merging hints from each hint provider.
        bestHint := TopologyHint{defaultAffinity, false}
        iterateAllProviderTopologyHints(filteredHints, func(permutation []TopologyHint) </span><span class="cov8" title="1">{
                // Get the NUMANodeAffinity from each hint in the permutation and see if any
                // of them encode unpreferred allocations.
                mergedHint := mergePermutation(numaNodes, permutation)
                // Only consider mergedHints that result in a NUMANodeAffinity &gt; 0 to
                // replace the current bestHint.
                if mergedHint.NUMANodeAffinity.Count() == 0 </span><span class="cov8" title="1">{
                        return
                }</span>

                // If the current bestHint is non-preferred and the new mergedHint is
                // preferred, always choose the preferred hint over the non-preferred one.
                <span class="cov8" title="1">if mergedHint.Preferred &amp;&amp; !bestHint.Preferred </span><span class="cov8" title="1">{
                        bestHint = mergedHint
                        return
                }</span>

                // If the current bestHint is preferred and the new mergedHint is
                // non-preferred, never update bestHint, regardless of mergedHint's
                // narowness.
                <span class="cov8" title="1">if !mergedHint.Preferred &amp;&amp; bestHint.Preferred </span><span class="cov8" title="1">{
                        return
                }</span>

                // If mergedHint and bestHint has the same preference, only consider
                // mergedHints that have a narrower NUMANodeAffinity than the
                // NUMANodeAffinity in the current bestHint.
                <span class="cov8" title="1">if !mergedHint.NUMANodeAffinity.IsNarrowerThan(bestHint.NUMANodeAffinity) </span><span class="cov8" title="1">{
                        return
                }</span>

                // In all other cases, update bestHint to the current mergedHint
                <span class="cov8" title="1">bestHint = mergedHint</span>
        })

        <span class="cov8" title="1">return bestHint</span>
}

// Iterate over all permutations of hints in 'allProviderHints [][]TopologyHint'.
//
// This procedure is implemented as a recursive function over the set of hints
// in 'allproviderHints[i]'. It applies the function 'callback' to each
// permutation as it is found. It is the equivalent of:
//
// for i := 0; i &lt; len(providerHints[0]); i++
//     for j := 0; j &lt; len(providerHints[1]); j++
//         for k := 0; k &lt; len(providerHints[2]); k++
//             ...
//             for z := 0; z &lt; len(providerHints[-1]); z++
//                 permutation := []TopologyHint{
//                     providerHints[0][i],
//                     providerHints[1][j],
//                     providerHints[2][k],
//                     ...
//                     providerHints[-1][z]
//                 }
//                 callback(permutation)
func iterateAllProviderTopologyHints(allProviderHints [][]TopologyHint, callback func([]TopologyHint)) <span class="cov8" title="1">{
        // Internal helper function to accumulate the permutation before calling the callback.
        var iterate func(i int, accum []TopologyHint)
        iterate = func(i int, accum []TopologyHint) </span><span class="cov8" title="1">{
                // Base case: we have looped through all providers and have a full permutation.
                if i == len(allProviderHints) </span><span class="cov8" title="1">{
                        callback(accum)
                        return
                }</span>

                // Loop through all hints for provider 'i', and recurse to build the
                // the permutation of this hint with all hints from providers 'i++'.
                <span class="cov8" title="1">for j := range allProviderHints[i] </span><span class="cov8" title="1">{
                        iterate(i+1, append(accum, allProviderHints[i][j]))
                }</span>
        }
        <span class="cov8" title="1">iterate(0, []TopologyHint{})</span>
}

// Merge a TopologyHints permutation to a single hint by performing a bitwise-AND
// of their affinity masks. The hint shall be preferred if all hits in the permutation
// are preferred.
func mergePermutation(numaNodes []int, permutation []TopologyHint) TopologyHint <span class="cov8" title="1">{
        // Get the NUMANodeAffinity from each hint in the permutation and see if any
        // of them encode unpreferred allocations.
        preferred := true
        defaultAffinity, _ := bitmask.NewBitMask(numaNodes...)
        var numaAffinities []bitmask.BitMask
        for _, hint := range permutation </span><span class="cov8" title="1">{
                // Only consider hints that have an actual NUMANodeAffinity set.
                if hint.NUMANodeAffinity == nil </span><span class="cov0" title="0">{
                        numaAffinities = append(numaAffinities, defaultAffinity)
                }</span> else<span class="cov8" title="1"> {
                        numaAffinities = append(numaAffinities, hint.NUMANodeAffinity)
                }</span>

                <span class="cov8" title="1">if !hint.Preferred </span><span class="cov8" title="1">{
                        preferred = false
                }</span>
        }

        // Merge the affinities using a bitwise-and operation.
        <span class="cov8" title="1">mergedAffinity := bitmask.And(defaultAffinity, numaAffinities...)
        // Build a mergedHint from the merged affinity mask, indicating if an
        // preferred allocation was used to generate the affinity mask or not.
        return TopologyHint{mergedAffinity, preferred}</span>
}
</pre>
		
		<pre class="file" id="file67" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import "k8s.io/klog"

type policyBestEffort struct {
        numaNodes []int
}

// NewPolicyBestEffort return a new policy interface
func NewPolicyBestEffort(numaNodes []int) Policy <span class="cov8" title="1">{
        return &amp;policyBestEffort{numaNodes: numaNodes}
}</span>

func (p *policyBestEffort) canAdmitPodResult(hint *TopologyHint) bool <span class="cov8" title="1">{
        return true
}</span>

func (p *policyBestEffort) Predicate(providersHints []map[string][]TopologyHint) (TopologyHint, bool) <span class="cov8" title="1">{
        filteredProvidersHints := filterProvidersHints(providersHints)
        bestHint := mergeFilteredHints(p.numaNodes, filteredProvidersHints)
        admit := p.canAdmitPodResult(&amp;bestHint)

        klog.V(4).Infof("bestHint: %v admit %v\n", bestHint, admit)
        return bestHint, admit
}</span>
</pre>
		
		<pre class="file" id="file68" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

type policyNone struct {
        numaNodes []int
}

// NewPolicyNone return a new policy interface
func NewPolicyNone(numaNodes []int) Policy <span class="cov0" title="0">{
        return &amp;policyNone{numaNodes: numaNodes}
}</span>

func (policy *policyNone) canAdmitPodResult(hint *TopologyHint) bool <span class="cov0" title="0">{
        return true
}</span>

func (policy *policyNone) Predicate(providersHints []map[string][]TopologyHint) (TopologyHint, bool) <span class="cov0" title="0">{
        return TopologyHint{}, policy.canAdmitPodResult(nil)
}</span>
</pre>
		
		<pre class="file" id="file69" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import "k8s.io/klog"

type policyRestricted struct {
        numaNodes []int
}

// NewPolicyRestricted return a new policy interface
func NewPolicyRestricted(numaNodes []int) Policy <span class="cov8" title="1">{
        return &amp;policyRestricted{numaNodes: numaNodes}
}</span>

func (p *policyRestricted) canAdmitPodResult(hint *TopologyHint) bool <span class="cov8" title="1">{
        return hint.Preferred
}</span>

func (p *policyRestricted) Predicate(providersHints []map[string][]TopologyHint) (TopologyHint, bool) <span class="cov8" title="1">{
        filteredHints := filterProvidersHints(providersHints)
        bestHint := mergeFilteredHints(p.numaNodes, filteredHints)
        admit := p.canAdmitPodResult(&amp;bestHint)

        klog.V(4).Infof("bestHint: %v admit %v\n", bestHint, admit)
        return bestHint, admit
}</span>
</pre>
		
		<pre class="file" id="file70" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import "k8s.io/klog"

type policySingleNumaNode struct {
        numaNodes []int
}

// NewPolicySingleNumaNode return a new policy interface
func NewPolicySingleNumaNode(numaNodes []int) Policy <span class="cov8" title="1">{
        return &amp;policySingleNumaNode{numaNodes: numaNodes}
}</span>

func (policy *policySingleNumaNode) canAdmitPodResult(hint *TopologyHint) bool <span class="cov8" title="1">{
        return hint.Preferred
}</span>

// Return hints that have valid bitmasks with exactly one bit set.
func filterSingleNumaHints(allResourcesHints [][]TopologyHint) [][]TopologyHint <span class="cov8" title="1">{
        var filteredResourcesHints [][]TopologyHint
        for _, oneResourceHints := range allResourcesHints </span><span class="cov8" title="1">{
                var filtered []TopologyHint
                for _, hint := range oneResourceHints </span><span class="cov8" title="1">{
                        if hint.NUMANodeAffinity == nil &amp;&amp; hint.Preferred </span><span class="cov0" title="0">{
                                filtered = append(filtered, hint)
                        }</span>
                        <span class="cov8" title="1">if hint.NUMANodeAffinity != nil &amp;&amp; hint.NUMANodeAffinity.Count() == 1 &amp;&amp; hint.Preferred </span><span class="cov8" title="1">{
                                filtered = append(filtered, hint)
                        }</span>
                }
                <span class="cov8" title="1">filteredResourcesHints = append(filteredResourcesHints, filtered)</span>
        }
        <span class="cov8" title="1">return filteredResourcesHints</span>
}

func (policy *policySingleNumaNode) Predicate(providersHints []map[string][]TopologyHint) (TopologyHint, bool) <span class="cov8" title="1">{
        filteredHints := filterProvidersHints(providersHints)
        singleNumaHints := filterSingleNumaHints(filteredHints)
        bestHint := mergeFilteredHints(policy.numaNodes, singleNumaHints)
        klog.V(4).Infof("bestHint: %v\n", bestHint)
        admit := policy.canAdmitPodResult(&amp;bestHint)
        return bestHint, admit
}</span>
</pre>
		
		<pre class="file" id="file71" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpumanager

import (
        "fmt"
        "sort"

        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
)

type cpuAccumulator struct {
        topo          *topology.CPUTopology
        details       topology.CPUDetails
        numCPUsNeeded int
        result        cpuset.CPUSet
}

func newCPUAccumulator(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int) *cpuAccumulator <span class="cov8" title="1">{
        return &amp;cpuAccumulator{
                topo:          topo,
                details:       topo.CPUDetails.KeepOnly(availableCPUs),
                numCPUsNeeded: numCPUs,
                result:        cpuset.NewCPUSet(),
        }
}</span>

func (a *cpuAccumulator) take(cpus cpuset.CPUSet) <span class="cov8" title="1">{
        a.result = a.result.Union(cpus)
        a.details = a.details.KeepOnly(a.details.CPUs().Difference(a.result))
        a.numCPUsNeeded -= cpus.Size()
}</span>

// isSocketFree Returns true if the supplied socket is fully available in `topoDetails`.
func (a *cpuAccumulator) isSocketFree(socketID int) bool <span class="cov8" title="1">{
        return a.details.CPUsInSockets(socketID).Size() == a.topo.CPUsPerSocket()
}</span>

// isCoreFree Returns true if the supplied core is fully available in `topoDetails`.
func (a *cpuAccumulator) isCoreFree(coreID int) bool <span class="cov8" title="1">{
        return a.details.CPUsInCores(coreID).Size() == a.topo.CPUsPerCore()
}</span>

// freeSockets Returns free socket IDs as a slice sorted by:
// - socket ID, ascending.
func (a *cpuAccumulator) freeSockets() []int <span class="cov8" title="1">{
        return a.details.Sockets().Filter(a.isSocketFree).ToSlice()
}</span>

// freeCores Returns core IDs as a slice sorted by:
// - the number of whole available cores on the socket, ascending
// - socket ID, ascending
// - core ID, ascending
func (a *cpuAccumulator) freeCores() []int <span class="cov8" title="1">{
        socketIDs := a.details.Sockets().ToSliceNoSort()
        sort.Slice(socketIDs,
                func(i, j int) bool </span><span class="cov0" title="0">{
                        iCores := a.details.CoresInSockets(socketIDs[i]).Filter(a.isCoreFree)
                        jCores := a.details.CoresInSockets(socketIDs[j]).Filter(a.isCoreFree)
                        return iCores.Size() &lt; jCores.Size() || socketIDs[i] &lt; socketIDs[j]
                }</span>)

        <span class="cov8" title="1">coreIDs := []int{}
        for _, s := range socketIDs </span><span class="cov8" title="1">{
                coreIDs = append(coreIDs, a.details.CoresInSockets(s).Filter(a.isCoreFree).ToSlice()...)
        }</span>
        <span class="cov8" title="1">return coreIDs</span>
}

// freeCPUs Returns CPU IDs as a slice sorted by:
// - socket affinity with result
// - number of CPUs available on the same socket
// - number of CPUs available on the same core
// - socket ID.
// - core ID.
func (a *cpuAccumulator) freeCPUs() []int <span class="cov8" title="1">{
        result := []int{}
        cores := a.details.Cores().ToSlice()

        sort.Slice(
                cores,
                func(i, j int) bool </span><span class="cov8" title="1">{
                        iCore := cores[i]
                        jCore := cores[j]

                        iCPUs := a.topo.CPUDetails.CPUsInCores(iCore).ToSlice()
                        jCPUs := a.topo.CPUDetails.CPUsInCores(jCore).ToSlice()

                        iSocket := a.topo.CPUDetails[iCPUs[0]].SocketID
                        jSocket := a.topo.CPUDetails[jCPUs[0]].SocketID

                        // Compute the number of CPUs in the result reside on the same socket
                        // as each core.
                        iSocketColoScore := a.topo.CPUDetails.CPUsInSockets(iSocket).Intersection(a.result).Size()
                        jSocketColoScore := a.topo.CPUDetails.CPUsInSockets(jSocket).Intersection(a.result).Size()

                        // Compute the number of available CPUs available on the same socket
                        // as each core.
                        iSocketFreeScore := a.details.CPUsInSockets(iSocket).Size()
                        jSocketFreeScore := a.details.CPUsInSockets(jSocket).Size()

                        // Compute the number of available CPUs on each core.
                        iCoreFreeScore := a.details.CPUsInCores(iCore).Size()
                        jCoreFreeScore := a.details.CPUsInCores(jCore).Size()

                        return iSocketColoScore &gt; jSocketColoScore ||
                                iSocketFreeScore &lt; jSocketFreeScore ||
                                iCoreFreeScore &lt; jCoreFreeScore ||
                                iSocket &lt; jSocket ||
                                iCore &lt; jCore
                }</span>)

        // For each core, append sorted CPU IDs to result.
        <span class="cov8" title="1">for _, core := range cores </span><span class="cov8" title="1">{
                result = append(result, a.details.CPUsInCores(core).ToSlice()...)
        }</span>
        <span class="cov8" title="1">return result</span>
}

func (a *cpuAccumulator) needs(n int) bool <span class="cov8" title="1">{
        return a.numCPUsNeeded &gt;= n
}</span>

func (a *cpuAccumulator) isSatisfied() bool <span class="cov8" title="1">{
        return a.numCPUsNeeded &lt; 1
}</span>

func (a *cpuAccumulator) isFailed() bool <span class="cov8" title="1">{
        return a.numCPUsNeeded &gt; a.details.CPUs().Size()
}</span>

// takeByTopology return the assigned cpuset
func takeByTopology(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int) (cpuset.CPUSet, error) <span class="cov8" title="1">{
        acc := newCPUAccumulator(topo, availableCPUs, numCPUs)
        if acc.isSatisfied() </span><span class="cov8" title="1">{
                return acc.result, nil
        }</span>
        <span class="cov8" title="1">if acc.isFailed() </span><span class="cov8" title="1">{
                return cpuset.NewCPUSet(), fmt.Errorf("not enough cpus available to satisfy request")
        }</span>

        // Algorithm: topology-aware best-fit
        // 1. Acquire whole sockets, if available and the container requires at
        //    least a socket's-worth of CPUs.
        <span class="cov8" title="1">if acc.needs(acc.topo.CPUsPerSocket()) </span><span class="cov8" title="1">{
                for _, s := range acc.freeSockets() </span><span class="cov8" title="1">{
                        klog.V(4).Infof("[cpumanager] takeByTopology: claiming socket [%d]", s)
                        acc.take(acc.details.CPUsInSockets(s))
                        if acc.isSatisfied() </span><span class="cov8" title="1">{
                                return acc.result, nil
                        }</span>
                        <span class="cov8" title="1">if !acc.needs(acc.topo.CPUsPerSocket()) </span><span class="cov8" title="1">{
                                break</span>
                        }
                }
        }

        // 2. Acquire whole cores, if available and the container requires at least
        //    a core's-worth of CPUs.
        <span class="cov8" title="1">if acc.needs(acc.topo.CPUsPerCore()) </span><span class="cov8" title="1">{
                for _, c := range acc.freeCores() </span><span class="cov8" title="1">{
                        klog.V(4).Infof("[cpumanager] takeByTopology: claiming core [%d]", c)
                        acc.take(acc.details.CPUsInCores(c))
                        if acc.isSatisfied() </span><span class="cov8" title="1">{
                                return acc.result, nil
                        }</span>
                        <span class="cov8" title="1">if !acc.needs(acc.topo.CPUsPerCore()) </span><span class="cov0" title="0">{
                                break</span>
                        }
                }
        }

        // 3. Acquire single threads, preferring to fill partially-allocated cores
        //    on the same sockets as the whole cores we have already taken in this
        //    allocation.
        <span class="cov8" title="1">for _, c := range acc.freeCPUs() </span><span class="cov8" title="1">{
                klog.V(4).Infof("[cpumanager] takeByTopology: claiming CPU [%d]", c)
                if acc.needs(1) </span><span class="cov8" title="1">{
                        acc.take(cpuset.NewCPUSet(c))
                }</span>
                <span class="cov8" title="1">if acc.isSatisfied() </span><span class="cov8" title="1">{
                        return acc.result, nil
                }</span>
        }

        <span class="cov0" title="0">return cpuset.NewCPUSet(), fmt.Errorf("failed to allocate cpus")</span>
}
</pre>
		
		<pre class="file" id="file72" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpumanager

import (
        "math"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
        "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/plugins/numaaware/policy"
)

type cpuMng struct {
}

// NewProvider return a new provider
func NewProvider() policy.HintProvider <span class="cov8" title="1">{
        return &amp;cpuMng{}
}</span>

// Name return the cpu manager name
func (mng *cpuMng) Name() string <span class="cov0" title="0">{
        return "cpuMng"
}</span>

// guaranteedCPUs return the intger num of request cpu
func guaranteedCPUs(container *v1.Container) int <span class="cov8" title="1">{
        cpuQuantity := container.Resources.Requests[v1.ResourceCPU]
        if cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() </span><span class="cov0" title="0">{
                return 0
        }</span>

        <span class="cov8" title="1">return int(cpuQuantity.Value())</span>
}

// generateCPUTopologyHints return the numa topology hints based on
// - availableCPUs
func generateCPUTopologyHints(availableCPUs cpuset.CPUSet, CPUDetails topology.CPUDetails, request int) []policy.TopologyHint <span class="cov8" title="1">{
        minAffinitySize := CPUDetails.NUMANodes().Size()
        hints := []policy.TopologyHint{}
        bitmask.IterateBitMasks(CPUDetails.NUMANodes().ToSlice(), func(mask bitmask.BitMask) </span><span class="cov8" title="1">{
                // First, update minAffinitySize for the current request size.
                cpusInMask := CPUDetails.CPUsInNUMANodes(mask.GetBits()...).Size()
                if cpusInMask &gt;= request &amp;&amp; mask.Count() &lt; minAffinitySize </span><span class="cov8" title="1">{
                        minAffinitySize = mask.Count()
                }</span>

                // Then check to see if we have enough CPUs available on the current
                // numa node bitmask to satisfy the CPU request.
                <span class="cov8" title="1">numMatching := 0
                // Finally, check to see if enough available CPUs remain on the current
                // NUMA node combination to satisfy the CPU request.
                for _, c := range availableCPUs.ToSlice() </span><span class="cov8" title="1">{
                        if mask.IsSet(CPUDetails[c].NUMANodeID) </span><span class="cov8" title="1">{
                                numMatching++
                        }</span>
                }

                // If they don't, then move onto the next combination.
                <span class="cov8" title="1">if numMatching &lt; request </span><span class="cov8" title="1">{
                        return
                }</span>

                // Otherwise, create a new hint from the numa node bitmask and add it to the
                // list of hints.  We set all hint preferences to 'false' on the first
                // pass through.
                <span class="cov8" title="1">hints = append(hints, policy.TopologyHint{
                        NUMANodeAffinity: mask,
                        Preferred:        false,
                })</span>
        })

        // Loop back through all hints and update the 'Preferred' field based on
        // counting the number of bits sets in the affinity mask and comparing it
        // to the minAffinitySize. Only those with an equal number of bits set (and
        // with a minimal set of numa nodes) will be considered preferred.
        <span class="cov8" title="1">for i := range hints </span><span class="cov8" title="1">{
                if hints[i].NUMANodeAffinity.Count() == minAffinitySize </span><span class="cov8" title="1">{
                        hints[i].Preferred = true
                }</span>
        }

        <span class="cov8" title="1">return hints</span>
}

func (mng *cpuMng) GetTopologyHints(container *v1.Container,
        topoInfo *api.NumatopoInfo, resNumaSets api.ResNumaSets) map[string][]policy.TopologyHint <span class="cov8" title="1">{
        if _, ok := container.Resources.Requests[v1.ResourceCPU]; !ok </span><span class="cov0" title="0">{
                klog.Warningf("container %s has no cpu request", container.Name)
                return nil
        }</span>

        <span class="cov8" title="1">requestNum := guaranteedCPUs(container)
        if requestNum == 0 </span><span class="cov0" title="0">{
                klog.Warningf(" the cpu request isn't  integer in container %s", container.Name)
                return nil
        }</span>

        <span class="cov8" title="1">cputopo := &amp;topology.CPUTopology{
                NumCPUs:    topoInfo.CPUDetail.CPUs().Size(),
                NumCores:   topoInfo.CPUDetail.Cores().Size() * topoInfo.CPUDetail.Sockets().Size(),
                NumSockets: topoInfo.CPUDetail.Sockets().Size(),
                CPUDetails: topoInfo.CPUDetail,
        }

        reserved := cpuset.NewCPUSet()
        reservedCPUs, ok := topoInfo.ResReserved[v1.ResourceCPU]
        if ok </span><span class="cov0" title="0">{
                // Take the ceiling of the reservation, since fractional CPUs cannot be
                // exclusively allocated.
                reservedCPUsFloat := float64(reservedCPUs.MilliValue()) / 1000
                numReservedCPUs := int(math.Ceil(reservedCPUsFloat))
                reserved, _ = takeByTopology(cputopo, cputopo.CPUDetails.CPUs(), numReservedCPUs)
                klog.V(4).Infof("[cpumanager] reserve cpuset :%v", reserved)
        }</span>

        <span class="cov8" title="1">availableCPUSet, ok := resNumaSets[string(v1.ResourceCPU)]
        if !ok </span><span class="cov0" title="0">{
                klog.Warningf("no cpu resource")
                return nil
        }</span>

        <span class="cov8" title="1">availableCPUSet = availableCPUSet.Difference(reserved)
        klog.V(4).Infof("requested: %d, availableCPUSet: %v", requestNum, availableCPUSet)
        return map[string][]policy.TopologyHint{
                string(v1.ResourceCPU): generateCPUTopologyHints(availableCPUSet, topoInfo.CPUDetail, requestNum),
        }</span>
}

func (mng *cpuMng) Allocate(container *v1.Container, bestHit *policy.TopologyHint,
        topoInfo *api.NumatopoInfo, resNumaSets api.ResNumaSets) map[string]cpuset.CPUSet <span class="cov8" title="1">{
        cputopo := &amp;topology.CPUTopology{
                NumCPUs:    topoInfo.CPUDetail.CPUs().Size(),
                NumCores:   topoInfo.CPUDetail.Cores().Size() * topoInfo.CPUDetail.Sockets().Size(),
                NumSockets: topoInfo.CPUDetail.Sockets().Size(),
                CPUDetails: topoInfo.CPUDetail,
        }

        reserved := cpuset.NewCPUSet()
        reservedCPUs, ok := topoInfo.ResReserved[v1.ResourceCPU]
        if ok </span><span class="cov0" title="0">{
                // Take the ceiling of the reservation, since fractional CPUs cannot be
                // exclusively allocated.
                reservedCPUsFloat := float64(reservedCPUs.MilliValue()) / 1000
                numReservedCPUs := int(math.Ceil(reservedCPUsFloat))
                reserved, _ = takeByTopology(cputopo, cputopo.CPUDetails.CPUs(), numReservedCPUs)
                klog.V(3).Infof("[cpumanager] reserve cpuset :%v", reserved)
        }</span>

        <span class="cov8" title="1">requestNum := guaranteedCPUs(container)
        availableCPUSet := resNumaSets[string(v1.ResourceCPU)]
        availableCPUSet = availableCPUSet.Difference(reserved)

        klog.V(4).Infof("alignedCPUs: %v requestNum: %v bestHit %v", availableCPUSet, requestNum, bestHit)

        result := cpuset.NewCPUSet()
        if bestHit.NUMANodeAffinity != nil </span><span class="cov8" title="1">{
                alignedCPUs := cpuset.NewCPUSet()
                for _, numaNodeID := range bestHit.NUMANodeAffinity.GetBits() </span><span class="cov8" title="1">{
                        alignedCPUs = alignedCPUs.Union(availableCPUSet.Intersection(cputopo.CPUDetails.CPUsInNUMANodes(numaNodeID)))
                }</span>

                <span class="cov8" title="1">numAlignedToAlloc := alignedCPUs.Size()
                if requestNum &lt; numAlignedToAlloc </span><span class="cov8" title="1">{
                        numAlignedToAlloc = requestNum
                }</span>

                <span class="cov8" title="1">alignedCPUs, err := takeByTopology(cputopo, alignedCPUs, numAlignedToAlloc)
                if err != nil </span><span class="cov0" title="0">{
                        return map[string]cpuset.CPUSet{
                                string(v1.ResourceCPU): cpuset.NewCPUSet(),
                        }
                }</span>

                <span class="cov8" title="1">result = result.Union(alignedCPUs)</span>
        }

        // Get any remaining CPUs from what's leftover after attempting to grab aligned ones.
        <span class="cov8" title="1">remainingCPUs, err := takeByTopology(cputopo, availableCPUSet.Difference(result), requestNum-result.Size())
        if err != nil </span><span class="cov8" title="1">{
                return map[string]cpuset.CPUSet{
                        string(v1.ResourceCPU): cpuset.NewCPUSet(),
                }
        }</span>

        <span class="cov8" title="1">result = result.Union(remainingCPUs)

        return map[string]cpuset.CPUSet{
                string(v1.ResourceCPU): result,
        }</span>
}
</pre>
		
		<pre class="file" id="file73" style="display: none">/*
Copyright 2020 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package predicates

import (
        "fmt"
        "sync"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
)

type predicateCache struct {
        sync.RWMutex
        cache map[string]map[string]bool //key_1: nodename key_2:pod uid
}

// predicateCacheNew return cache map
func predicateCacheNew() *predicateCache <span class="cov8" title="1">{
        return &amp;predicateCache{
                cache: make(map[string]map[string]bool),
        }
}</span>

// getPodTemplateUID return pod template key
func getPodTemplateUID(pod *v1.Pod) string <span class="cov0" title="0">{
        uid, found := pod.Annotations[batch.PodTemplateKey]
        if !found </span><span class="cov0" title="0">{
                return ""
        }</span>

        <span class="cov0" title="0">return uid</span>
}

// PredicateWithCache: check the predicate result existed in cache
func (pc *predicateCache) PredicateWithCache(nodeName string, pod *v1.Pod) (bool, error) <span class="cov0" title="0">{
        podTemplateUID := getPodTemplateUID(pod)
        if podTemplateUID == "" </span><span class="cov0" title="0">{
                return false, fmt.Errorf("no anonation of volcano.sh/template-uid in pod %s", pod.Name)
        }</span>

        <span class="cov0" title="0">pc.RLock()
        defer pc.RUnlock()
        if nodeCache, exist := pc.cache[nodeName]; exist </span><span class="cov0" title="0">{
                if result, exist := nodeCache[podTemplateUID]; exist </span><span class="cov0" title="0">{
                        klog.V(4).Infof("Predicate node %s and pod %s result %v", nodeName, pod.Name, result)
                        return result, nil
                }</span>
        }

        <span class="cov0" title="0">return false, fmt.Errorf("no information of node %s and pod %s in predicate cache", nodeName, pod.Name)</span>
}

// UpdateCache update cache data
func (pc *predicateCache) UpdateCache(nodeName string, pod *v1.Pod, fit bool) <span class="cov0" title="0">{
        podTemplateUID := getPodTemplateUID(pod)
        if podTemplateUID == "" </span><span class="cov0" title="0">{
                klog.V(3).Infof("Don't find pod %s template uid", pod.Name)
                return
        }</span>

        <span class="cov0" title="0">pc.Lock()
        defer pc.Unlock()

        if _, exist := pc.cache[nodeName]; !exist </span><span class="cov0" title="0">{
                podCache := make(map[string]bool)
                podCache[podTemplateUID] = fit
                pc.cache[nodeName] = podCache
        }</span> else<span class="cov0" title="0"> {
                pc.cache[nodeName][podTemplateUID] = fit
        }</span>
}
</pre>
		
		<pre class="file" id="file74" style="display: none">/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package predicates

import (
        "fmt"

        v1 "k8s.io/api/core/v1"

        "volcano.sh/volcano/pkg/scheduler/api"
)

// checkNodeGPUSharingPredicate checks if a gpu sharing pod can be scheduled on a node.
func checkNodeGPUSharingPredicate(pod *v1.Pod, nodeInfo *api.NodeInfo) (bool, error) <span class="cov0" title="0">{
        // no gpu sharing request
        if api.GetGPUResourceOfPod(pod) &lt;= 0 </span><span class="cov0" title="0">{
                return true, nil
        }</span>

        <span class="cov0" title="0">id := predicateGPU(pod, nodeInfo)
        if id &lt; 0 </span><span class="cov0" title="0">{
                return false, fmt.Errorf("no enough gpu memory on single device of node %s", nodeInfo.Name)
        }</span>
        <span class="cov0" title="0">return true, nil</span>
}

// predicateGPU returns the available GPU ID
func predicateGPU(pod *v1.Pod, node *api.NodeInfo) int <span class="cov0" title="0">{
        gpuRequest := api.GetGPUResourceOfPod(pod)
        allocatableGPUs := node.GetDevicesIdleGPUMemory()

        for devID := 0; devID &lt; len(allocatableGPUs); devID++ </span><span class="cov0" title="0">{
                availableGPU, ok := allocatableGPUs[devID]
                if ok </span><span class="cov0" title="0">{
                        if availableGPU &gt;= gpuRequest </span><span class="cov0" title="0">{
                                return devID
                        }</span>
                }
        }

        <span class="cov0" title="0">return -1</span>
}
</pre>
		
		<pre class="file" id="file75" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package predicates

import (
        "context"
        "fmt"
        "strings"

        v1 "k8s.io/api/core/v1"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/scheduler/apis/config"
        "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity"
        "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity"
        "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports"
        "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable"
        "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration"
        k8sframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/plugins/util"
        "volcano.sh/volcano/pkg/scheduler/plugins/util/k8s"
)

const (
        // PluginName indicates name of volcano scheduler plugin.
        PluginName = "predicates"

        // GPUSharingPredicate is the key for enabling GPU Sharing Predicate in YAML
        GPUSharingPredicate = "predicate.GPUSharingEnable"

        // CachePredicate control cache predicate feature
        CachePredicate = "predicate.CacheEnable"

        // ProportionalPredicate is the key for enabling Proportional Predicate in YAML
        ProportionalPredicate = "predicate.ProportionalEnable"
        // ProportionalResource is the key for additional resource key name
        ProportionalResource = "predicate.resources"
        // ProportionalResourcesPrefix is the key prefix for additional resource key name
        ProportionalResourcesPrefix = ProportionalResource + "."
)

type predicatesPlugin struct {
        // Arguments given for the plugin
        pluginArguments framework.Arguments
}

// New return predicate plugin
func New(arguments framework.Arguments) framework.Plugin <span class="cov8" title="1">{
        return &amp;predicatesPlugin{pluginArguments: arguments}
}</span>

func (pp *predicatesPlugin) Name() string <span class="cov8" title="1">{
        return PluginName
}</span>

type baseResource struct {
        CPU    float64
        Memory float64
}

type predicateEnable struct {
        gpuSharingEnable   bool
        cacheEnable        bool
        proportionalEnable bool
        proportional       map[v1.ResourceName]baseResource
}

func enablePredicate(args framework.Arguments) predicateEnable <span class="cov8" title="1">{
        /*
           User Should give predicatesEnable in this format(predicate.GPUSharingEnable).
           Currently supported only GPUSharing predicate checks.

           actions: "reclaim, allocate, backfill, preempt"
           tiers:
           - plugins:
             - name: priority
             - name: gang
             - name: conformance
           - plugins:
             - name: drf
             - name: predicates
               arguments:
                 predicate.GPUSharingEnable: true
                 predicate.CacheEnable: true
                 predicate.ProportionalEnable: true
                 predicate.resources: nvidia.com/gpu
                 predicate.resources.nvidia.com/gpu.cpu: 4
                 predicate.resources.nvidia.com/gpu.memory: 8
             - name: proportion
             - name: nodeorder
        */

        predicate := predicateEnable{
                gpuSharingEnable:   false,
                cacheEnable:        false,
                proportionalEnable: false,
        }

        // Checks whether predicate.GPUSharingEnable is provided or not, if given, modifies the value in predicateEnable struct.
        args.GetBool(&amp;predicate.gpuSharingEnable, GPUSharingPredicate)
        args.GetBool(&amp;predicate.cacheEnable, CachePredicate)
        // Checks whether predicate.ProportionalEnable is provided or not, if given, modifies the value in predicateEnable struct.
        args.GetBool(&amp;predicate.proportionalEnable, ProportionalPredicate)
        resourcesProportional := make(map[v1.ResourceName]baseResource)
        resourcesStr := args[ProportionalResource]
        resources := strings.Split(resourcesStr, ",")
        for _, resource := range resources </span><span class="cov8" title="1">{
                resource = strings.TrimSpace(resource)
                if resource == "" </span><span class="cov8" title="1">{
                        continue</span>
                }
                // proportional.resources.[ResourceName]
                <span class="cov0" title="0">cpuResourceKey := ProportionalResourcesPrefix + resource + ".cpu"
                cpuResourceRate := 1.0
                args.GetFloat64(&amp;cpuResourceRate, cpuResourceKey)
                if cpuResourceRate &lt; 0 </span><span class="cov0" title="0">{
                        cpuResourceRate = 1.0
                }</span>
                <span class="cov0" title="0">memoryResourceKey := ProportionalResourcesPrefix + resource + ".memory"
                memoryResourceRate := 1.0
                args.GetFloat64(&amp;memoryResourceRate, memoryResourceKey)
                if memoryResourceRate &lt; 0 </span><span class="cov0" title="0">{
                        memoryResourceRate = 1.0
                }</span>
                <span class="cov0" title="0">r := baseResource{
                        CPU:    cpuResourceRate,
                        Memory: memoryResourceRate,
                }
                resourcesProportional[v1.ResourceName(resource)] = r</span>
        }
        <span class="cov8" title="1">predicate.proportional = resourcesProportional

        return predicate</span>
}

func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) <span class="cov8" title="1">{
        pl := util.NewPodListerFromNode(ssn)
        nodeMap := util.GenerateNodeMapAndSlice(ssn.Nodes)

        pCache := predicateCacheNew()
        predicate := enablePredicate(pp.pluginArguments)

        kubeClient := ssn.KubeClient()
        // Register event handlers to update task info in PodLister &amp; nodeMap
        ssn.AddEventHandler(&amp;framework.EventHandler{
                AllocateFunc: func(event *framework.Event) </span><span class="cov8" title="1">{
                        pod := pl.UpdateTask(event.Task, event.Task.NodeName)

                        nodeName := event.Task.NodeName
                        node, found := nodeMap[nodeName]
                        if !found </span><span class="cov0" title="0">{
                                klog.Errorf("predicates, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName)
                                return
                        }</span>

                        <span class="cov8" title="1">if predicate.gpuSharingEnable &amp;&amp; api.GetGPUResourceOfPod(pod) &gt; 0 </span><span class="cov0" title="0">{
                                nodeInfo, ok := ssn.Nodes[nodeName]
                                if !ok </span><span class="cov0" title="0">{
                                        klog.Errorf("Failed to get node %s info from cache", nodeName)
                                        return
                                }</span>

                                <span class="cov0" title="0">id := predicateGPU(pod, nodeInfo)
                                if id &lt; 0 </span><span class="cov0" title="0">{
                                        klog.Errorf("The node %s can't place the pod %s in ns %s", pod.Spec.NodeName, pod.Name, pod.Namespace)
                                        return
                                }</span>
                                <span class="cov0" title="0">patch := api.AddGPUIndexPatch(id)
                                pod, err := kubeClient.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
                                if err != nil </span><span class="cov0" title="0">{
                                        klog.Errorf("Patch pod %s failed with patch %s: %v", pod.Name, patch, err)
                                        return
                                }</span>
                                <span class="cov0" title="0">dev, ok := nodeInfo.GPUDevices[id]
                                if !ok </span><span class="cov0" title="0">{
                                        klog.Errorf("Failed to get GPU %d from node %s", id, nodeName)
                                        return
                                }</span>
                                <span class="cov0" title="0">dev.PodMap[string(pod.UID)] = pod
                                klog.V(4).Infof("predicates with gpu sharing, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName)</span>
                        }

                        <span class="cov8" title="1">node.AddPod(pod)
                        klog.V(4).Infof("predicates, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName)</span>
                },
                DeallocateFunc: func(event *framework.Event) <span class="cov8" title="1">{
                        pod := pl.UpdateTask(event.Task, "")
                        nodeName := event.Task.NodeName
                        node, found := nodeMap[nodeName]
                        if !found </span><span class="cov0" title="0">{
                                klog.Errorf("predicates, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName)
                                return
                        }</span>

                        <span class="cov8" title="1">if predicate.gpuSharingEnable &amp;&amp; api.GetGPUResourceOfPod(pod) &gt; 0 </span><span class="cov0" title="0">{
                                // deallocate pod gpu id
                                id := api.GetGPUIndex(pod)
                                patch := api.RemoveGPUIndexPatch()
                                _, err := kubeClient.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
                                if err != nil </span><span class="cov0" title="0">{
                                        klog.Errorf("Patch pod %s failed with patch %s: %v", pod.Name, patch, err)
                                        return
                                }</span>

                                <span class="cov0" title="0">nodeInfo, ok := ssn.Nodes[nodeName]
                                if !ok </span><span class="cov0" title="0">{
                                        klog.Errorf("Failed to get node %s info from cache", nodeName)
                                        return
                                }</span>
                                <span class="cov0" title="0">if dev, ok := nodeInfo.GPUDevices[id]; ok </span><span class="cov0" title="0">{
                                        delete(dev.PodMap, string(pod.UID))
                                }</span>

                                <span class="cov0" title="0">klog.V(4).Infof("predicates with gpu sharing, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName)</span>
                        }

                        <span class="cov8" title="1">err := node.RemovePod(pod)
                        if err != nil </span><span class="cov0" title="0">{
                                klog.Errorf("predicates, remove pod %s/%s from node [%s] error: %v", pod.Namespace, pod.Name, nodeName, err)
                                return
                        }</span>
                        <span class="cov8" title="1">klog.V(4).Infof("predicates, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName)</span>
                },
        })

        // Initialize k8s plugins
        // TODO: Add more predicates, k8s.io/kubernetes/pkg/scheduler/framework/plugins/legacy_registry.go
        <span class="cov8" title="1">handle := k8s.NewFrameworkHandle(nodeMap, ssn.KubeClient(), ssn.InformerFactory())
        // 1. NodeUnschedulable
        plugin, _ := nodeunschedulable.New(nil, handle)
        nodeUnscheduleFilter := plugin.(*nodeunschedulable.NodeUnschedulable)
        // 2. NodeAffinity
        plugin, _ = nodeaffinity.New(nil, handle)
        nodeAffinityFilter := plugin.(*nodeaffinity.NodeAffinity)
        // 3. NodePorts
        plugin, _ = nodeports.New(nil, handle)
        nodePortFilter := plugin.(*nodeports.NodePorts)
        // 4. TaintToleration
        plugin, _ = tainttoleration.New(nil, handle)
        tolerationFilter := plugin.(*tainttoleration.TaintToleration)
        // 5. InterPodAffinity
        plArgs := &amp;config.InterPodAffinityArgs{}
        plugin, _ = interpodaffinity.New(plArgs, handle)
        podAffinityFilter := plugin.(*interpodaffinity.InterPodAffinity)

        ssn.AddPredicateFn(pp.Name(), func(task *api.TaskInfo, node *api.NodeInfo) error </span><span class="cov8" title="1">{
                nodeInfo, found := nodeMap[node.Name]
                if !found </span><span class="cov0" title="0">{
                        return fmt.Errorf("failed to predicates, node info for %s not found", node.Name)
                }</span>

                <span class="cov8" title="1">if node.Allocatable.MaxTaskNum &lt;= len(nodeInfo.Pods) </span><span class="cov0" title="0">{
                        klog.V(4).Infof("NodePodNumber predicates Task &lt;%s/%s&gt; on Node &lt;%s&gt; failed",
                                task.Namespace, task.Name, node.Name)
                        return api.NewFitError(task, node, api.NodePodNumberExceeded)
                }</span>

                <span class="cov8" title="1">state := k8sframework.NewCycleState()
                predicateByStablefilter := func(pod *v1.Pod, nodeInfo *k8sframework.NodeInfo) (bool, error) </span><span class="cov8" title="1">{
                        // CheckNodeUnschedulable
                        status := nodeUnscheduleFilter.Filter(context.TODO(), state, task.Pod, nodeInfo)
                        if !status.IsSuccess() </span><span class="cov0" title="0">{
                                return false, fmt.Errorf("plugin %s predicates failed %s", nodeunschedulable.Name, status.Message())
                        }</span>

                        // Check NodeAffinity
                        <span class="cov8" title="1">status = nodeAffinityFilter.Filter(context.TODO(), state, task.Pod, nodeInfo)
                        if !status.IsSuccess() </span><span class="cov8" title="1">{
                                return false, fmt.Errorf("plugin %s predicates failed %s", nodeaffinity.Name, status.Message())
                        }</span>

                        // PodToleratesNodeTaints: TaintToleration
                        <span class="cov8" title="1">status = tolerationFilter.Filter(context.TODO(), state, task.Pod, nodeInfo)
                        if !status.IsSuccess() </span><span class="cov0" title="0">{
                                return false, fmt.Errorf("plugin %s predicates failed %s", tainttoleration.Name, status.Message())
                        }</span>

                        <span class="cov8" title="1">return true, nil</span>
                }

                // Check PredicateWithCache
                <span class="cov8" title="1">{
                        var err error
                        var fit bool
                        if predicate.cacheEnable </span><span class="cov0" title="0">{
                                fit, err = pCache.PredicateWithCache(node.Name, task.Pod)
                                if err != nil </span><span class="cov0" title="0">{
                                        fit, err = predicateByStablefilter(task.Pod, nodeInfo)
                                        pCache.UpdateCache(node.Name, task.Pod, fit)
                                }</span> else<span class="cov0" title="0"> {
                                        if !fit </span><span class="cov0" title="0">{
                                                err = fmt.Errorf("plugin equivalence cache predicates failed")
                                        }</span>
                                }
                        } else<span class="cov8" title="1"> {
                                fit, err = predicateByStablefilter(task.Pod, nodeInfo)
                        }</span>

                        <span class="cov8" title="1">if !fit </span><span class="cov8" title="1">{
                                return err
                        }</span>
                }

                // Check NodePorts
                <span class="cov8" title="1">nodePortFilter.PreFilter(context.TODO(), state, task.Pod)
                status := nodePortFilter.Filter(context.TODO(), state, nil, nodeInfo)
                if !status.IsSuccess() </span><span class="cov0" title="0">{
                        return fmt.Errorf("plugin %s predicates failed %s", nodeaffinity.Name, status.Message())
                }</span>

                // InterPodAffinity Predicate
                <span class="cov8" title="1">status = podAffinityFilter.PreFilter(context.TODO(), state, task.Pod)
                if !status.IsSuccess() </span><span class="cov0" title="0">{
                        return fmt.Errorf("plugin %s pre-predicates failed %s", interpodaffinity.Name, status.Message())
                }</span>

                <span class="cov8" title="1">status = podAffinityFilter.Filter(context.TODO(), state, task.Pod, nodeInfo)
                if !status.IsSuccess() </span><span class="cov0" title="0">{
                        return fmt.Errorf("plugin %s predicates failed %s", interpodaffinity.Name, status.Message())
                }</span>

                <span class="cov8" title="1">if predicate.gpuSharingEnable </span><span class="cov0" title="0">{
                        // CheckGPUSharingPredicate
                        fit, err := checkNodeGPUSharingPredicate(task.Pod, node)
                        if err != nil </span><span class="cov0" title="0">{
                                return err
                        }</span>

                        <span class="cov0" title="0">klog.V(4).Infof("checkNodeGPUSharingPredicate predicates Task &lt;%s/%s&gt; on Node &lt;%s&gt;: fit %v",
                                task.Namespace, task.Name, node.Name, fit)</span>
                }
                <span class="cov8" title="1">if predicate.proportionalEnable </span><span class="cov0" title="0">{
                        // Check ProportionalPredicate
                        fit, err := checkNodeResourceIsProportional(task, node, predicate.proportional)
                        if err != nil </span><span class="cov0" title="0">{
                                return err
                        }</span>
                        <span class="cov0" title="0">klog.V(4).Infof("checkNodeResourceIsProportional predicates Task &lt;%s/%s&gt; on Node &lt;%s&gt;: fit %v",
                                task.Namespace, task.Name, node.Name, fit)</span>
                }
                <span class="cov8" title="1">return nil</span>
        })
}

func (pp *predicatesPlugin) OnSessionClose(ssn *framework.Session) {<span class="cov8" title="1">}</span>
</pre>
		
		<pre class="file" id="file76" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package predicates

import (
        "fmt"

        v1 "k8s.io/api/core/v1"

        "volcano.sh/volcano/pkg/scheduler/api"
)

// checkNodeResourceIsProportional checks if a gpu:cpu:memory is Proportional
func checkNodeResourceIsProportional(task *api.TaskInfo, node *api.NodeInfo, proportional map[v1.ResourceName]baseResource) (bool, error) <span class="cov8" title="1">{
        for resourceName := range proportional </span><span class="cov8" title="1">{
                if value, found := task.Resreq.ScalarResources[resourceName]; found &amp;&amp; value &gt; 0 </span><span class="cov8" title="1">{
                        return true, nil
                }</span>
        }
        <span class="cov8" title="1">for resourceName, resourceRate := range proportional </span><span class="cov8" title="1">{
                if value, found := node.Idle.ScalarResources[resourceName]; found </span><span class="cov8" title="1">{
                        cpuReserved := value * resourceRate.CPU
                        memoryReserved := value * resourceRate.Memory * 1000 * 1000
                        r := node.Idle.Clone()
                        r = r.Sub(task.Resreq)
                        if r.MilliCPU &lt; cpuReserved || r.Memory &lt; memoryReserved </span><span class="cov8" title="1">{
                                return false, fmt.Errorf("proportional of resource %s check failed", resourceName)
                        }</span>
                }
        }
        <span class="cov8" title="1">return true, nil</span>
}
</pre>
		
		<pre class="file" id="file77" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tasktopology

import (
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
)

type reqAction int

const (
        reqSub reqAction = iota
        reqAdd
)

// Bucket is struct used to classify tasks by affinity and anti-affinity
type Bucket struct {
        index       int
        tasks       map[types.UID]*api.TaskInfo
        taskNameSet map[string]int

        // reqScore is score of resource
        // now, we regard 1 CPU and 1 GPU and 1Gi memory as the same score.
        reqScore float64
        request  *api.Resource

        boundTask int
        node      map[string]int
}

// NewBucket create a new empty bucket
func NewBucket() *Bucket <span class="cov0" title="0">{
        return &amp;Bucket{
                index:       0,
                tasks:       make(map[types.UID]*api.TaskInfo),
                taskNameSet: make(map[string]int),

                reqScore: 0,
                request:  api.EmptyResource(),

                boundTask: 0,
                node:      make(map[string]int),
        }
}</span>

// CalcResReq calculates task resources request
func (b *Bucket) CalcResReq(req *api.Resource, action reqAction) <span class="cov0" title="0">{
        if req == nil </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov0" title="0">cpu := req.MilliCPU
        // treat 1Mi the same as 1m cpu 1m gpu
        mem := req.Memory / 1024 / 1024
        score := cpu + mem
        for _, request := range req.ScalarResources </span><span class="cov0" title="0">{
                score += request
        }</span>

        <span class="cov0" title="0">switch action </span>{
        case reqSub:<span class="cov0" title="0">
                b.reqScore -= score
                b.request.Sub(req)</span>
        case reqAdd:<span class="cov0" title="0">
                b.reqScore += score
                b.request.Add(req)</span>
        default:<span class="cov0" title="0">
                klog.V(3).Infof("Invalid action &lt;%v&gt; for resource &lt;%v&gt;", action, req)</span>
        }
}

// AddTask adds task into bucket
func (b *Bucket) AddTask(taskName string, task *api.TaskInfo) <span class="cov0" title="0">{
        b.taskNameSet[taskName]++
        if task.NodeName != "" </span><span class="cov0" title="0">{
                b.node[task.NodeName]++
                b.boundTask++
                return
        }</span>

        <span class="cov0" title="0">b.tasks[task.Pod.UID] = task
        b.CalcResReq(task.Resreq, reqAdd)</span>
}

// TaskBound binds task to bucket
func (b *Bucket) TaskBound(task *api.TaskInfo) <span class="cov0" title="0">{
        b.node[task.NodeName]++
        b.boundTask++

        delete(b.tasks, task.Pod.UID)
        b.CalcResReq(task.Resreq, reqSub)
}</span>
</pre>
		
		<pre class="file" id="file78" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tasktopology

import (
        "fmt"
        "math"
        "sort"
        "strings"

        "k8s.io/apimachinery/pkg/types"
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
)

type topologyType int

const (
        selfAntiAffinity topologyType = iota
        interAntiAffinity
        selfAffinity
        interAffinity
)

// map[topologyType]priority, the larger number means the higher priority
var affinityPriority = map[topologyType]int{
        selfAntiAffinity:  4,
        interAffinity:     3,
        selfAffinity:      2,
        interAntiAffinity: 1,
}

// JobManager is struct used to save infos about affinity and buckets of a job
type JobManager struct {
        jobID api.JobID

        buckets     []*Bucket
        podInBucket map[types.UID]int
        podInTask   map[types.UID]string
        taskOverPod map[string]map[types.UID]struct{}

        taskAffinityPriority map[string]int // [taskName] -&gt; priority
        taskExistOrder       map[string]int
        interAffinity        map[string]map[string]struct{} // [taskName]-&gt;[taskName]
        selfAffinity         map[string]struct{}
        interAntiAffinity    map[string]map[string]struct{} // [taskName]-&gt;[taskName]
        selfAntiAffinity     map[string]struct{}

        bucketMaxSize int
        nodeTaskSet   map[string]map[string]int // [nodeName]-&gt;[taskName]
}

// NewJobManager creates a new job manager for job
func NewJobManager(jobID api.JobID) *JobManager <span class="cov0" title="0">{
        return &amp;JobManager{
                jobID: jobID,

                buckets:     make([]*Bucket, 0),
                podInBucket: make(map[types.UID]int),
                podInTask:   make(map[types.UID]string),
                taskOverPod: make(map[string]map[types.UID]struct{}),

                taskAffinityPriority: make(map[string]int),
                taskExistOrder:       make(map[string]int),
                interAffinity:        make(map[string]map[string]struct{}),
                interAntiAffinity:    make(map[string]map[string]struct{}),
                selfAffinity:         make(map[string]struct{}),
                selfAntiAffinity:     make(map[string]struct{}),

                bucketMaxSize: 0,
                nodeTaskSet:   make(map[string]map[string]int),
        }
}</span>

// MarkOutOfBucket indicates task is outside of any bucket
func (jm *JobManager) MarkOutOfBucket(uid types.UID) <span class="cov0" title="0">{
        jm.podInBucket[uid] = OutOfBucket
}</span>

// MarkTaskHasTopology indicates task has topology settings
func (jm *JobManager) MarkTaskHasTopology(taskName string, topoType topologyType) <span class="cov0" title="0">{
        priority := affinityPriority[topoType]
        if priority &gt; jm.taskAffinityPriority[taskName] </span><span class="cov0" title="0">{
                jm.taskAffinityPriority[taskName] = priority
        }</span>
}

// ApplyTaskTopology transforms taskTopology to matrix
// affinity: [[a, b], [c]]
// interAffinity:
//      a   b   c
//  a   -   x   -
//  b   x   -   -
//  c   -   -   -
//  selfAffinity:
//      a   b   c
//      -   -   x
func (jm *JobManager) ApplyTaskTopology(topo *TaskTopology) <span class="cov0" title="0">{
        for _, aff := range topo.Affinity </span><span class="cov0" title="0">{
                if len(aff) == 1 </span><span class="cov0" title="0">{
                        taskName := aff[0]
                        jm.selfAffinity[taskName] = struct{}{}
                        jm.MarkTaskHasTopology(taskName, selfAffinity)
                        continue</span>
                }
                <span class="cov0" title="0">for index, src := range aff </span><span class="cov0" title="0">{
                        for _, dst := range aff[:index] </span><span class="cov0" title="0">{
                                addAffinity(jm.interAffinity, src, dst)
                                addAffinity(jm.interAffinity, dst, src)
                        }</span>
                        <span class="cov0" title="0">jm.MarkTaskHasTopology(src, interAffinity)</span>
                }
        }

        <span class="cov0" title="0">for _, aff := range topo.AntiAffinity </span><span class="cov0" title="0">{
                if len(aff) == 1 </span><span class="cov0" title="0">{
                        taskName := aff[0]
                        jm.selfAntiAffinity[taskName] = struct{}{}
                        jm.MarkTaskHasTopology(taskName, selfAntiAffinity)
                        continue</span>
                }
                <span class="cov0" title="0">for index, src := range aff </span><span class="cov0" title="0">{
                        for _, dst := range aff[:index] </span><span class="cov0" title="0">{
                                addAffinity(jm.interAntiAffinity, src, dst)
                                addAffinity(jm.interAntiAffinity, dst, src)
                        }</span>
                        <span class="cov0" title="0">jm.MarkTaskHasTopology(src, interAntiAffinity)</span>
                }
        }

        <span class="cov0" title="0">length := len(topo.TaskOrder)
        for index, taskName := range topo.TaskOrder </span><span class="cov0" title="0">{
                jm.taskExistOrder[taskName] = length - index
        }</span>
}

// NewBucket creates a new bucket
func (jm *JobManager) NewBucket() *Bucket <span class="cov0" title="0">{
        bucket := NewBucket()
        bucket.index = len(jm.buckets)
        jm.buckets = append(jm.buckets, bucket)
        return bucket
}</span>

// AddTaskToBucket adds task into bucket
func (jm *JobManager) AddTaskToBucket(bucketIndex int, taskName string, task *api.TaskInfo) <span class="cov0" title="0">{
        bucket := jm.buckets[bucketIndex]
        jm.podInBucket[task.Pod.UID] = bucketIndex
        bucket.AddTask(taskName, task)
        if size := len(bucket.tasks) + bucket.boundTask; size &gt; jm.bucketMaxSize </span><span class="cov0" title="0">{
                jm.bucketMaxSize = size
        }</span>
}

// L compared with R, -1 for L &lt; R, 0 for L == R, 1 for L &gt; R
func (jm *JobManager) taskAffinityOrder(L, R *api.TaskInfo) int <span class="cov0" title="0">{
        LTaskName := jm.podInTask[L.Pod.UID]
        RTaskName := jm.podInTask[R.Pod.UID]

        // in the same vk task, they are equal
        if LTaskName == RTaskName </span><span class="cov0" title="0">{
                return 0
        }</span>

        // use user defined order firstly
        <span class="cov0" title="0">LOrder := jm.taskExistOrder[LTaskName]
        ROrder := jm.taskExistOrder[RTaskName]
        if LOrder != ROrder </span><span class="cov0" title="0">{
                if LOrder &gt; ROrder </span><span class="cov0" title="0">{
                        return 1
                }</span>
                <span class="cov0" title="0">return -1</span>
        }

        <span class="cov0" title="0">LPriority := jm.taskAffinityPriority[LTaskName]
        RPriority := jm.taskAffinityPriority[RTaskName]
        if LPriority != RPriority </span><span class="cov0" title="0">{
                if LPriority &gt; RPriority </span><span class="cov0" title="0">{
                        return 1
                }</span>
                <span class="cov0" title="0">return -1</span>
        }

        // all affinity setting of L and R are the same, they are equal
        <span class="cov0" title="0">return 0</span>
}

func (jm *JobManager) buildTaskInfo(tasks map[api.TaskID]*api.TaskInfo) []*api.TaskInfo <span class="cov0" title="0">{
        taskWithoutBucket := make([]*api.TaskInfo, 0, len(tasks))
        for _, task := range tasks </span><span class="cov0" title="0">{
                pod := task.Pod

                taskName := getTaskName(task)
                if taskName == "" </span><span class="cov0" title="0">{
                        jm.MarkOutOfBucket(pod.UID)
                        continue</span>
                }
                <span class="cov0" title="0">if _, hasTopology := jm.taskAffinityPriority[taskName]; !hasTopology </span><span class="cov0" title="0">{
                        jm.MarkOutOfBucket(pod.UID)
                        continue</span>
                }

                <span class="cov0" title="0">jm.podInTask[pod.UID] = taskName
                taskSet, ok := jm.taskOverPod[taskName]
                if !ok </span><span class="cov0" title="0">{
                        taskSet = make(map[types.UID]struct{})
                        jm.taskOverPod[taskName] = taskSet
                }</span>
                <span class="cov0" title="0">taskSet[pod.UID] = struct{}{}
                taskWithoutBucket = append(taskWithoutBucket, task)</span>
        }
        <span class="cov0" title="0">return taskWithoutBucket</span>
}

func (jm *JobManager) checkTaskSetAffinity(taskName string, taskNameSet map[string]int, onlyAnti bool) int <span class="cov0" title="0">{
        bucketPodAff := 0

        if taskName == "" </span><span class="cov0" title="0">{
                return bucketPodAff
        }</span>

        <span class="cov0" title="0">for taskNameInBucket, count := range taskNameSet </span><span class="cov0" title="0">{
                theSameTask := taskNameInBucket == taskName

                if !onlyAnti </span><span class="cov0" title="0">{
                        affinity := false
                        if theSameTask </span><span class="cov0" title="0">{
                                _, affinity = jm.selfAffinity[taskName]
                        }</span> else<span class="cov0" title="0"> {
                                _, affinity = jm.interAffinity[taskName][taskNameInBucket]
                        }</span>
                        <span class="cov0" title="0">if affinity </span><span class="cov0" title="0">{
                                bucketPodAff += count
                        }</span>
                }

                <span class="cov0" title="0">antiAffinity := false
                if theSameTask </span><span class="cov0" title="0">{
                        _, antiAffinity = jm.selfAntiAffinity[taskName]
                }</span> else<span class="cov0" title="0"> {
                        _, antiAffinity = jm.interAntiAffinity[taskName][taskNameInBucket]
                }</span>
                <span class="cov0" title="0">if antiAffinity </span><span class="cov0" title="0">{
                        bucketPodAff -= count
                }</span>
        }

        <span class="cov0" title="0">return bucketPodAff</span>
}

func (jm *JobManager) buildBucket(taskWithOrder []*api.TaskInfo) <span class="cov0" title="0">{
        nodeBucketMapping := make(map[string]*Bucket)

        for _, task := range taskWithOrder </span><span class="cov0" title="0">{
                klog.V(5).Infof("jobID %s task with order task %s/%s", jm.jobID, task.Namespace, task.Name)

                var selectedBucket *Bucket
                maxAffinity := math.MinInt32

                taskName := getTaskName(task)

                if task.NodeName != "" </span><span class="cov0" title="0">{
                        // generate bucket by node
                        maxAffinity = 0
                        selectedBucket = nodeBucketMapping[task.NodeName]
                }</span> else<span class="cov0" title="0"> {
                        for _, bucket := range jm.buckets </span><span class="cov0" title="0">{
                                bucketPodAff := jm.checkTaskSetAffinity(taskName, bucket.taskNameSet, false)

                                // choose the best fit affinity, or balance resource between bucket
                                if bucketPodAff &gt; maxAffinity </span><span class="cov0" title="0">{
                                        maxAffinity = bucketPodAff
                                        selectedBucket = bucket
                                }</span> else<span class="cov0" title="0"> if bucketPodAff == maxAffinity &amp;&amp; selectedBucket != nil &amp;&amp;
                                        bucket.reqScore &lt; selectedBucket.reqScore </span><span class="cov0" title="0">{
                                        selectedBucket = bucket
                                }</span>
                        }
                }

                <span class="cov0" title="0">if maxAffinity &lt; 0 || selectedBucket == nil </span><span class="cov0" title="0">{
                        selectedBucket = jm.NewBucket()
                        if task.NodeName != "" </span><span class="cov0" title="0">{
                                nodeBucketMapping[task.NodeName] = selectedBucket
                        }</span>
                }

                <span class="cov0" title="0">jm.AddTaskToBucket(selectedBucket.index, taskName, task)</span>
        }
}

// ConstructBucket builds bucket for tasks
func (jm *JobManager) ConstructBucket(tasks map[api.TaskID]*api.TaskInfo) <span class="cov0" title="0">{
        taskWithoutBucket := jm.buildTaskInfo(tasks)

        o := TaskOrder{
                tasks: taskWithoutBucket,

                manager: jm,
        }
        sort.Sort(sort.Reverse(&amp;o))

        jm.buildBucket(o.tasks)
}</span>

// TaskBound binds task to bucket
func (jm *JobManager) TaskBound(task *api.TaskInfo) <span class="cov0" title="0">{
        if taskName := getTaskName(task); taskName != "" </span><span class="cov0" title="0">{
                set, ok := jm.nodeTaskSet[task.NodeName]
                if !ok </span><span class="cov0" title="0">{
                        set = make(map[string]int)
                        jm.nodeTaskSet[task.NodeName] = set
                }</span>
                <span class="cov0" title="0">set[taskName]++</span>
        }

        <span class="cov0" title="0">bucket := jm.GetBucket(task)
        if bucket != nil </span><span class="cov0" title="0">{
                bucket.TaskBound(task)
        }</span>
}

// GetBucket get bucket inside which task has been
func (jm *JobManager) GetBucket(task *api.TaskInfo) *Bucket <span class="cov0" title="0">{
        index, ok := jm.podInBucket[task.Pod.UID]
        if !ok || index == OutOfBucket </span><span class="cov0" title="0">{
                return nil
        }</span>

        <span class="cov0" title="0">bucket := jm.buckets[index]
        return bucket</span>
}

func (jm *JobManager) String() string <span class="cov0" title="0">{
        // saa: selfAntiAffinity
        // iaa: interAntiAffinity
        // sa: selfAffinity
        // ia: interAffinity
        msg := []string{
                fmt.Sprintf("%s - job %s max %d || saa: %v - iaa: %v - sa: %v - ia: %v || priority: %v - order: %v || ",
                        PluginName, jm.jobID, jm.bucketMaxSize,
                        jm.selfAntiAffinity, jm.interAntiAffinity,
                        jm.selfAffinity, jm.interAffinity,
                        jm.taskAffinityPriority, jm.taskExistOrder,
                ),
        }

        for _, bucket := range jm.buckets </span><span class="cov0" title="0">{
                bucketMsg := fmt.Sprintf("b:%d -- ", bucket.index)
                var info []string
                for _, task := range bucket.tasks </span><span class="cov0" title="0">{
                        info = append(info, task.Pod.Name)
                }</span>
                <span class="cov0" title="0">bucketMsg += strings.Join(info, ", ")
                bucketMsg += "|"

                info = nil
                for nodeName, count := range bucket.node </span><span class="cov0" title="0">{
                        info = append(info, fmt.Sprintf("n%s-%d", nodeName, count))
                }</span>
                <span class="cov0" title="0">bucketMsg += strings.Join(info, ", ")

                msg = append(msg, "["+bucketMsg+"]")</span>
        }
        <span class="cov0" title="0">return strings.Join(msg, " ")</span>
}
</pre>
		
		<pre class="file" id="file79" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tasktopology

import (
        "fmt"
        "strings"
        "time"

        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
)

type taskTopologyPlugin struct {
        arguments framework.Arguments

        weight   int
        managers map[api.JobID]*JobManager
}

// New function returns taskTopologyPlugin object
func New(arguments framework.Arguments) framework.Plugin <span class="cov0" title="0">{
        return &amp;taskTopologyPlugin{
                arguments: arguments,

                weight:   calculateWeight(arguments),
                managers: make(map[api.JobID]*JobManager),
        }
}</span>

func (p *taskTopologyPlugin) Name() string <span class="cov0" title="0">{
        return PluginName
}</span>

// TaskOrderFn returns -1 to make l prior to r.
//
// for example:
// A:
//  | bucket1   | bucket2   | out of bucket
//  | a1 a3     | a2        | a4
// B:
//  | bucket1   | out of bucket
//  | b1 b2     | b3
// the right task order should be:
//   a1 a3 a2 b1 b2 a4 b3
func (p *taskTopologyPlugin) TaskOrderFn(l interface{}, r interface{}) int <span class="cov0" title="0">{
        lv, ok := l.(*api.TaskInfo)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Object is not a taskinfo")
        }</span>
        <span class="cov0" title="0">rv, ok := r.(*api.TaskInfo)
        if !ok </span><span class="cov0" title="0">{
                klog.Errorf("Object is not a taskinfo")
        }</span>

        <span class="cov0" title="0">lvJobManager := p.managers[lv.Job]
        rvJobManager := p.managers[rv.Job]

        var lvBucket, rvBucket *Bucket
        if lvJobManager != nil </span><span class="cov0" title="0">{
                lvBucket = lvJobManager.GetBucket(lv)
        }</span> else<span class="cov0" title="0"> {
                klog.V(4).Infof("No job manager for job &lt;ID: %s&gt;, do not return task order.", lv.Job)
                return 0
        }</span>
        <span class="cov0" title="0">if rvJobManager != nil </span><span class="cov0" title="0">{
                rvBucket = rvJobManager.GetBucket(rv)
        }</span> else<span class="cov0" title="0"> {
                klog.V(4).Infof("No job manager for job &lt;ID: %s&gt;, do not return task order.", rv.Job)
                return 0
        }</span>

        // the one have bucket would always prior to another
        <span class="cov0" title="0">lvInBucket := lvBucket != nil
        rvInBucket := rvBucket != nil
        if lvInBucket != rvInBucket </span><span class="cov0" title="0">{
                if lvInBucket </span><span class="cov0" title="0">{
                        return -1
                }</span>
                <span class="cov0" title="0">return 1</span>
        }

        // comparison between job is not the duty of this plugin
        <span class="cov0" title="0">if lv.Job != rv.Job </span><span class="cov0" title="0">{
                return 0
        }</span>

        // task out of bucket have no order
        <span class="cov0" title="0">if !lvInBucket &amp;&amp; !rvInBucket </span><span class="cov0" title="0">{
                return 0
        }</span>

        // the big bucket should prior to small one
        <span class="cov0" title="0">lvHasTask := len(lvBucket.tasks)
        rvHasTask := len(rvBucket.tasks)
        if lvHasTask != rvHasTask </span><span class="cov0" title="0">{
                if lvHasTask &gt; rvHasTask </span><span class="cov0" title="0">{
                        return -1
                }</span>
                <span class="cov0" title="0">return 1</span>
        }

        <span class="cov0" title="0">lvBucketIndex := lvBucket.index
        rvBucketIndex := rvBucket.index
        // in the same bucket, the affinityOrder is ok
        if lvBucketIndex == rvBucketIndex </span><span class="cov0" title="0">{
                affinityOrder := lvJobManager.taskAffinityOrder(lv, rv)
                return -affinityOrder
        }</span>

        // the old bucket should prior to young one
        <span class="cov0" title="0">if lvBucketIndex &lt; rvBucketIndex </span><span class="cov0" title="0">{
                return -1
        }</span>
        <span class="cov0" title="0">return 1</span>
}

func (p *taskTopologyPlugin) calcBucketScore(task *api.TaskInfo, node *api.NodeInfo) (int, *JobManager, error) <span class="cov0" title="0">{
        // task could never fits the node
        maxResource := node.Idle.Clone().Add(node.Releasing)
        if req := task.Resreq; req != nil &amp;&amp; maxResource.LessPartly(req, api.Zero) </span><span class="cov0" title="0">{
                return 0, nil, nil
        }</span>

        <span class="cov0" title="0">jobManager, hasManager := p.managers[task.Job]
        if !hasManager </span><span class="cov0" title="0">{
                return 0, nil, nil
        }</span>

        <span class="cov0" title="0">bucket := jobManager.GetBucket(task)
        // task out of bucket
        if bucket == nil </span><span class="cov0" title="0">{
                return 0, jobManager, nil
        }</span>

        // 1. bound task in bucket is the base score of this node
        <span class="cov0" title="0">score := bucket.node[node.Name]

        // 2. task inter/self anti-affinity should be calculated
        if nodeTaskSet := jobManager.nodeTaskSet[node.Name]; nodeTaskSet != nil </span><span class="cov0" title="0">{
                taskName := getTaskName(task)
                affinityScore := jobManager.checkTaskSetAffinity(taskName, nodeTaskSet, true)
                if affinityScore &lt; 0 </span><span class="cov0" title="0">{
                        score += affinityScore
                }</span>
        }
        <span class="cov0" title="0">klog.V(4).Infof("task %s/%s, node %s, additional score %d, task %d",
                task.Namespace, task.Name, node.Name, score, len(bucket.tasks))

        // 3. the other tasks in bucket take into considering
        score += len(bucket.tasks)
        if bucket.request == nil || bucket.request.LessEqual(maxResource, api.Zero) </span><span class="cov0" title="0">{
                return score, jobManager, nil
        }</span>

        <span class="cov0" title="0">remains := bucket.request.Clone()
        // randomly (by map) take out task to make the bucket fits the node
        for bucketTaskID, bucketTask := range bucket.tasks </span><span class="cov0" title="0">{
                // current task should kept in bucket
                if bucketTaskID == task.Pod.UID || bucketTask.Resreq == nil </span><span class="cov0" title="0">{
                        continue</span>
                }
                <span class="cov0" title="0">remains.Sub(bucketTask.Resreq)
                score--
                if remains.LessEqual(maxResource, api.Zero) </span><span class="cov0" title="0">{
                        break</span>
                }
        }
        // here, the bucket remained request will always fit the maxResource
        <span class="cov0" title="0">return score, jobManager, nil</span>
}

func (p *taskTopologyPlugin) NodeOrderFn(task *api.TaskInfo, node *api.NodeInfo) (float64, error) <span class="cov0" title="0">{
        score, jobManager, err := p.calcBucketScore(task, node)
        if err != nil </span><span class="cov0" title="0">{
                return 0, err
        }</span>
        <span class="cov0" title="0">fScore := float64(score * p.weight)
        if jobManager != nil &amp;&amp; jobManager.bucketMaxSize != 0 </span><span class="cov0" title="0">{
                fScore = fScore * float64(v1alpha1.MaxNodeScore) / float64(jobManager.bucketMaxSize)
        }</span>
        <span class="cov0" title="0">klog.V(4).Infof("task %s/%s at node %s has bucket score %d, score %f",
                task.Namespace, task.Name, node.Name, score, fScore)
        return fScore, nil</span>
}

func (p *taskTopologyPlugin) AllocateFunc(event *framework.Event) <span class="cov0" title="0">{
        task := event.Task

        jobManager, hasManager := p.managers[task.Job]
        if !hasManager </span><span class="cov0" title="0">{
                return
        }</span>
        <span class="cov0" title="0">jobManager.TaskBound(task)</span>
}

func (p *taskTopologyPlugin) initBucket(ssn *framework.Session) <span class="cov0" title="0">{
        for jobID, job := range ssn.Jobs </span><span class="cov0" title="0">{
                if noPendingTasks(job) </span><span class="cov0" title="0">{
                        klog.V(4).Infof("No pending tasks in job &lt;%s/%s&gt; by plugin %s.",
                                job.Namespace, job.Name, PluginName)
                        continue</span>
                }

                <span class="cov0" title="0">jobTopology, err := readTopologyFromPgAnnotations(job)
                if err != nil </span><span class="cov0" title="0">{
                        klog.V(4).Infof("Failed to read task topology from job &lt;%s/%s&gt; annotations, error: %s.",
                                job.Namespace, job.Name, err.Error())
                        continue</span>
                }
                <span class="cov0" title="0">if jobTopology == nil </span><span class="cov0" title="0">{
                        continue</span>
                }

                <span class="cov0" title="0">manager := NewJobManager(jobID)
                manager.ApplyTaskTopology(jobTopology)
                manager.ConstructBucket(job.Tasks)

                p.managers[job.UID] = manager</span>
        }
}

func affinityCheck(job *api.JobInfo, affinity [][]string) error <span class="cov8" title="1">{
        if job == nil || affinity == nil </span><span class="cov0" title="0">{
                return fmt.Errorf("empty input, job: %v, affinity: %v", job, affinity)
        }</span>

        <span class="cov8" title="1">var taskNumber = len(job.Tasks)
        var taskRef = make(map[string]bool, taskNumber)
        for _, task := range job.Tasks </span><span class="cov8" title="1">{
                tmpStrings := strings.Split(task.Name, "-")
                if _, exist := taskRef[tmpStrings[len(tmpStrings)-2]]; !exist </span><span class="cov8" title="1">{
                        taskRef[tmpStrings[len(tmpStrings)-2]] = true
                }</span>
        }

        <span class="cov8" title="1">for _, aff := range affinity </span><span class="cov8" title="1">{
                affTasks := make(map[string]bool, len(aff))
                for _, task := range aff </span><span class="cov8" title="1">{
                        if len(task) == 0 </span><span class="cov8" title="1">{
                                continue</span>
                        }
                        <span class="cov8" title="1">if _, exist := taskRef[task]; !exist </span><span class="cov8" title="1">{
                                return fmt.Errorf("task %s do not exist in job &lt;%s/%s&gt;", task, job.Namespace, job.Name)
                        }</span>
                        <span class="cov8" title="1">if _, exist := affTasks[task]; exist </span><span class="cov8" title="1">{
                                return fmt.Errorf("task %s is duplicated in job &lt;%s/%s&gt;", task, job.Namespace, job.Name)
                        }</span>
                        <span class="cov8" title="1">affTasks[task] = true</span>
                }
        }

        <span class="cov8" title="1">return nil</span>
}

func splitAnnotations(job *api.JobInfo, annotation string) ([][]string, error) <span class="cov8" title="1">{
        affinityStr := strings.Split(annotation, ";")
        if len(affinityStr) == 0 </span><span class="cov0" title="0">{
                return nil, nil
        }</span>
        <span class="cov8" title="1">var affinity = make([][]string, len(affinityStr))
        for i, str := range affinityStr </span><span class="cov8" title="1">{
                affinity[i] = strings.Split(str, ",")
        }</span>
        <span class="cov8" title="1">if err := affinityCheck(job, affinity); err != nil </span><span class="cov8" title="1">{
                klog.V(4).Infof("Job &lt;%s/%s&gt; affinity key invalid: %s.",
                        job.Namespace, job.Name, err.Error())
                return nil, err
        }</span>
        <span class="cov8" title="1">return affinity, nil</span>
}

func readTopologyFromPgAnnotations(job *api.JobInfo) (*TaskTopology, error) <span class="cov8" title="1">{
        jobAffinityStr, affinityExist := job.PodGroup.Annotations[JobAffinityAnnotations]
        jobAntiAffinityStr, antiAffinityExist := job.PodGroup.Annotations[JobAntiAffinityAnnotations]
        taskOrderStr, taskOrderExist := job.PodGroup.Annotations[TaskOrderAnnotations]

        if !(affinityExist || antiAffinityExist || taskOrderExist) </span><span class="cov8" title="1">{
                return nil, nil
        }</span>

        <span class="cov8" title="1">var jobTopology = TaskTopology{
                Affinity:     nil,
                AntiAffinity: nil,
                TaskOrder:    nil,
        }

        if affinityExist </span><span class="cov8" title="1">{
                affinities, err := splitAnnotations(job, jobAffinityStr)
                if err != nil </span><span class="cov8" title="1">{
                        klog.V(4).Infof("Job &lt;%s/%s&gt; affinity key invalid: %s.",
                                job.Namespace, job.Name, err.Error())
                        return nil, err
                }</span>
                <span class="cov8" title="1">jobTopology.Affinity = affinities</span>
        }

        <span class="cov8" title="1">if antiAffinityExist </span><span class="cov8" title="1">{
                affinities, err := splitAnnotations(job, jobAntiAffinityStr)
                if err != nil </span><span class="cov8" title="1">{
                        klog.V(4).Infof("Job &lt;%s/%s&gt; anti affinity key invalid: %s.",
                                job.Namespace, job.Name, err.Error())
                        return nil, err
                }</span>
                <span class="cov8" title="1">jobTopology.AntiAffinity = affinities</span>
        }

        <span class="cov8" title="1">if taskOrderExist </span><span class="cov8" title="1">{
                jobTopology.TaskOrder = strings.Split(taskOrderStr, ",")
                if err := affinityCheck(job, [][]string{jobTopology.TaskOrder}); err != nil </span><span class="cov8" title="1">{
                        klog.V(4).Infof("Job &lt;%s/%s&gt; task order key invalid: %s.",
                                job.Namespace, job.Name, err.Error())
                        return nil, err
                }</span>
        }

        <span class="cov8" title="1">return &amp;jobTopology, nil</span>
}

func (p *taskTopologyPlugin) OnSessionOpen(ssn *framework.Session) <span class="cov0" title="0">{
        start := time.Now()
        klog.V(3).Infof("start to init task topology plugin, weight[%d], defined order %v", p.weight, affinityPriority)

        p.initBucket(ssn)

        ssn.AddTaskOrderFn(p.Name(), p.TaskOrderFn)

        ssn.AddNodeOrderFn(p.Name(), p.NodeOrderFn)

        ssn.AddEventHandler(&amp;framework.EventHandler{
                AllocateFunc: p.AllocateFunc,
        })

        klog.V(3).Infof("finished to init task topology plugin, using time %v", time.Since(start))
}</span>

func (p *taskTopologyPlugin) OnSessionClose(ssn *framework.Session) <span class="cov0" title="0">{
        p.managers = nil
}</span>
</pre>
		
		<pre class="file" id="file80" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tasktopology

import (
        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
)

const (
        // PluginName indicates name of volcano scheduler plugin
        PluginName = "task-topology"
        // PluginWeight is task-topology plugin weight in nodeOrderFn
        PluginWeight = "task-topology.weight"
        // JobAffinityKey is the key to read in task-topology arguments from job annotations
        JobAffinityKey = "volcano.sh/task-topology"
        // OutOfBucket indicates task is outside of any bucket
        OutOfBucket = -1

        // JobAffinityAnnotations is the key to read in task-topology affinity arguments from podgroup annotations
        JobAffinityAnnotations = "volcano.sh/task-topology-affinity"
        // JobAntiAffinityAnnotations is the key to read in task-topology anti-affinity arguments from podgroup annotations
        JobAntiAffinityAnnotations = "volcano.sh/task-topology-anti-affinity"
        // TaskOrderAnnotations is the key to read in task-topology task order arguments from podgroup annotations
        TaskOrderAnnotations = "volcano.sh/task-topology-task-order"
)

// TaskTopology is struct used to save affinity infos of a job read from job plugin or annotations
type TaskTopology struct {
        Affinity     [][]string `json:"affinity,omitempty"`
        AntiAffinity [][]string `json:"antiAffinity,omitempty"`
        TaskOrder    []string   `json:"taskOrder,omitempty"`
}

func calculateWeight(args framework.Arguments) int <span class="cov0" title="0">{
        /*
           User Should give taskTopologyWeight in this format(task-topology.weight).

           actions: "enqueue, reclaim, allocate, backfill, preempt"
           tiers:
           - plugins:
             - name: task-topology
               arguments:
                 task-topology.weight: 10
        */
        // Values are initialized to 1.
        weight := 1

        args.GetInt(&amp;weight, PluginWeight)

        return weight
}</span>

func getTaskName(task *api.TaskInfo) string <span class="cov0" title="0">{
        return task.Pod.Annotations[v1alpha1.TaskSpecKey]
}</span>

func addAffinity(m map[string]map[string]struct{}, src, dst string) <span class="cov0" title="0">{
        srcMap, ok := m[src]
        if !ok </span><span class="cov0" title="0">{
                srcMap = make(map[string]struct{})
                m[src] = srcMap
        }</span>
        <span class="cov0" title="0">srcMap[dst] = struct{}{}</span>
}

func noPendingTasks(job *api.JobInfo) bool <span class="cov0" title="0">{
        return len(job.TaskStatusIndex[api.Pending]) == 0
}</span>

// TaskOrder is struct used to save task order
type TaskOrder struct {
        tasks   []*api.TaskInfo
        manager *JobManager
}

func (p *TaskOrder) Len() int <span class="cov0" title="0">{ return len(p.tasks) }</span>

func (p *TaskOrder) Swap(l, r int) <span class="cov0" title="0">{
        p.tasks[l], p.tasks[r] = p.tasks[r], p.tasks[l]
}</span>

func (p *TaskOrder) Less(l, r int) bool <span class="cov0" title="0">{
        L := p.tasks[l]
        R := p.tasks[r]

        LHasNode := L.NodeName != ""
        RHasNode := R.NodeName != ""
        if LHasNode || RHasNode </span><span class="cov0" title="0">{
                // the task bounded would have high priority
                if LHasNode != RHasNode </span><span class="cov0" title="0">{
                        return !LHasNode
                }</span>
                // all bound, any order is alright
                <span class="cov0" title="0">return L.NodeName &gt; R.NodeName</span>
        }

        <span class="cov0" title="0">result := p.manager.taskAffinityOrder(L, R)
        // they have the same taskAffinity order, any order is alright
        if result == 0 </span><span class="cov0" title="0">{
                return L.Name &gt; R.Name
        }</span>
        <span class="cov0" title="0">return result &lt; 0</span>
}
</pre>
		
		<pre class="file" id="file81" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tdm

import (
        "fmt"
        "strings"
        "time"

        "k8s.io/apimachinery/pkg/util/intstr"
        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
        tutil "volcano.sh/volcano/pkg/scheduler/plugins/util"
        "volcano.sh/volcano/pkg/scheduler/util"
)

const (
        // PluginName indicates name of volcano scheduler plugin.
        PluginName = "tdm"
        // revocableZoneLayout revocable zone layout
        revocableZoneLayout      = "15:04"
        revocableZoneLabelPrefix = "tdm.revocable-zone."
        evictPeriodLabel         = "tdm.evict.period"
        defaultPodEvictNum       = 1
)

var lastEvictAt time.Time

/*
   actions: "enqueue, reclaim, allocate, preempt"
   tiers:
   - plugins:
     - name: tdm
       arguments:
         tdm.revocable-zone.rz1: 10:00-21:00
         tdm.revocable-zone.rz2: 12:00-14:00
         tdm.evict.period: 1m
*/

type tdmPlugin struct {
        revocableZone map[string]string
        // evictPeriod
        // default 1m
        evictPeriod time.Duration
}

// New function returns prioritizePlugin object
func New(args framework.Arguments) framework.Plugin <span class="cov8" title="1">{
        revocableZone := make(map[string]string)
        evictPeriod := time.Minute

        for k, v := range args </span><span class="cov8" title="1">{
                if strings.Contains(k, revocableZoneLabelPrefix) </span><span class="cov8" title="1">{
                        revocableZone[strings.Replace(k, revocableZoneLabelPrefix, "", 1)] = v
                }</span>
        }

        <span class="cov8" title="1">if period, ok := args[evictPeriodLabel]; ok </span><span class="cov8" title="1">{
                if d, err := time.ParseDuration(period); err == nil </span><span class="cov8" title="1">{
                        evictPeriod = d
                }</span>
        }

        <span class="cov8" title="1">return &amp;tdmPlugin{revocableZone, evictPeriod}</span>
}

func (tp *tdmPlugin) Name() string <span class="cov8" title="1">{
        return PluginName
}</span>

func parseRevocableZone(rzRaw string) (start, end time.Time, err error) <span class="cov8" title="1">{
        rzValues := strings.Split(strings.TrimSpace(rzRaw), "-")

        if len(rzValues) != 2 </span><span class="cov8" title="1">{
                err = fmt.Errorf("revocable zone %v format error", rzRaw)
                return
        }</span>

        <span class="cov8" title="1">t1, err := time.Parse(revocableZoneLayout, rzValues[0])
        if err != nil </span><span class="cov8" title="1">{
                return
        }</span>

        <span class="cov8" title="1">t2, err := time.Parse(revocableZoneLayout, rzValues[1])
        if err != nil </span><span class="cov0" title="0">{
                return
        }</span>

        <span class="cov8" title="1">now := time.Now()

        start = time.Date(now.Year(), now.Month(), now.Day(), t1.Hour(), t1.Minute(), 0, 0, now.Location())
        if t1.After(t2) || t1.Equal(t2) </span><span class="cov8" title="1">{
                end = time.Date(now.Year(), now.Month(), now.Day()+1, t2.Hour(), t2.Minute(), 0, 0, now.Location())
        }</span> else<span class="cov8" title="1"> {
                end = time.Date(now.Year(), now.Month(), now.Day(), t2.Hour(), t2.Minute(), 0, 0, now.Location())
        }</span>

        <span class="cov8" title="1">return</span>
}

func (tp *tdmPlugin) availableRevocableZone(rz string) error <span class="cov8" title="1">{
        // rzRaw format 00:00-23:59
        rzRaw, ok := tp.revocableZone[rz]
        if !ok </span><span class="cov0" title="0">{
                return fmt.Errorf("revocable zone %v not support", rz)
        }</span>

        <span class="cov8" title="1">now := time.Now()

        start, end, err := parseRevocableZone(rzRaw)
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if now.Unix() &lt; start.Unix() || now.Unix() &gt; end.Unix() </span><span class="cov8" title="1">{
                return fmt.Errorf("current time beyond revocable zone %v:%v", rz, rzRaw)
        }</span>

        <span class="cov8" title="1">return nil</span>
}

func (tp *tdmPlugin) OnSessionOpen(ssn *framework.Session) <span class="cov8" title="1">{
        klog.V(4).Infof("Enter tdm plugin ...")
        if klog.V(4) </span><span class="cov0" title="0">{
                defer func() </span><span class="cov0" title="0">{
                        klog.V(4).Infof("Leaving tdm plugin.")
                }</span>()
        }

        // tdm plugin just handle revocable node
        <span class="cov8" title="1">predicateFn := func(task *api.TaskInfo, node *api.NodeInfo) error </span><span class="cov8" title="1">{
                if node.RevocableZone == "" </span><span class="cov8" title="1">{
                        return nil
                }</span>

                <span class="cov8" title="1">if err := tp.availableRevocableZone(node.RevocableZone); err != nil </span><span class="cov8" title="1">{
                        return fmt.Errorf("plugin %s predicates %w", tp.Name(), err)
                }</span>

                <span class="cov8" title="1">klog.V(4).Infof("TDM node %v revocable zone %v:%v is active", node.Name, node.RevocableZone, tp.revocableZone[node.RevocableZone])

                if len(task.RevocableZone) == 0 </span><span class="cov8" title="1">{
                        msg := fmt.Sprintf("task %s/%s is not allow to dispatch to revocable node %s", task.Namespace, task.Name, node.Name)
                        return fmt.Errorf("plugin %s predicates %s", tp.Name(), msg)
                }</span>

                <span class="cov8" title="1">klog.V(4).Infof("TDM filter for Task %s/%s on node %s pass.", task.Namespace, task.Name, node.Name)
                return nil</span>
        }

        // tdm plugin just handle revocable node
        <span class="cov8" title="1">nodeOrderFn := func(task *api.TaskInfo, node *api.NodeInfo) (float64, error) </span><span class="cov8" title="1">{
                score := 0.0

                if node.RevocableZone == "" </span><span class="cov8" title="1">{
                        return score, nil
                }</span>

                <span class="cov8" title="1">if err := tp.availableRevocableZone(node.RevocableZone); err != nil </span><span class="cov0" title="0">{
                        klog.V(4).Infof("TDM not available %s", err)
                        return score, err
                }</span>

                <span class="cov8" title="1">if len(task.RevocableZone) == 0 </span><span class="cov0" title="0">{
                        klog.V(4).Infof("TDM task %s/%s is not allow to dispatch to revocable node %s", task.Namespace, task.Name, node.Name)
                        return score, nil
                }</span>

                <span class="cov8" title="1">score = float64(v1alpha1.MaxNodeScore)

                klog.V(4).Infof("TDM score for Task %s/%s on node %s is: %v", task.Namespace, task.Name, node.Name, score)
                return score, nil</span>
        }

        <span class="cov8" title="1">preemptableFn := func(preemptor *api.TaskInfo, preemptees []*api.TaskInfo) ([]*api.TaskInfo, int) </span><span class="cov0" title="0">{
                // for the preemptable or can use revocablezone workload, they can not preempt other tasks.
                if preemptor.Preemptable || len(preemptor.RevocableZone) &gt; 0 </span><span class="cov0" title="0">{
                        klog.V(4).Infof("TDM task %s/%s is preemptable, do nothing skip", preemptor.Namespace, preemptor.Name)
                        return nil, tutil.Reject
                }</span>

                <span class="cov0" title="0">var victims []*api.TaskInfo
                tasksMap := make(map[api.JobID][]*api.TaskInfo)

                // find preemptable tasks which appear on none revocable node
                for _, task := range preemptees </span><span class="cov0" title="0">{
                        if !task.Preemptable || task.Status != api.Running </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov0" title="0">node, ok := ssn.Nodes[task.NodeName]
                        if !ok </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov0" title="0">if node.RevocableZone != "" </span><span class="cov0" title="0">{
                                continue</span>
                        }

                        <span class="cov0" title="0">tasksMap[task.Job] = append(tasksMap[task.Job], task)</span>
                }

                <span class="cov0" title="0">for jobID, preemptableTasks := range tasksMap </span><span class="cov0" title="0">{
                        if job, ok := ssn.Jobs[jobID]; ok </span><span class="cov0" title="0">{
                                victims = append(victims, tp.maxVictims(job, preemptableTasks)...)
                        }</span>
                }

                <span class="cov0" title="0">klog.V(4).Infof("TDM victims are %+v", victims)

                return victims, tutil.Permit</span>
        }

        <span class="cov8" title="1">victimsFn := func() []*api.TaskInfo </span><span class="cov8" title="1">{
                if lastEvictAt.Add(tp.evictPeriod).After(time.Now()) </span><span class="cov0" title="0">{
                        klog.V(4).Infof("TDM next evict time at %v", lastEvictAt)
                        return nil
                }</span>

                <span class="cov8" title="1">klog.V(4).Infof("TDM start to find victims")

                // find preemptable task on timeout revocable zone node
                victims := make([]*api.TaskInfo, 0)
                for rz := range tp.revocableZone </span><span class="cov8" title="1">{
                        if err := tp.availableRevocableZone(rz); err != nil </span><span class="cov8" title="1">{
                                klog.V(4).Infof("TDM revocable zone %v disactive, %v", rz, err)
                                // rz disactive, then evict preemptable tasks by job from the revocable node
                                for jobID, preemtableTasks := range tp.revocableNodePreemptableTask(rz, ssn) </span><span class="cov8" title="1">{
                                        if job, ok := ssn.Jobs[jobID]; ok </span><span class="cov8" title="1">{
                                                victims = append(victims, tp.maxVictims(job, preemtableTasks)...)
                                        }</span>
                                }
                        }
                }

                // need to consider concurrency?
                <span class="cov8" title="1">lastEvictAt = time.Now()

                klog.V(4).Infof("TDM got %v victims", len(victims))

                return victims</span>
        }

        <span class="cov8" title="1">jobOrderFn := func(l, r interface{}) int </span><span class="cov0" title="0">{
                lv := l.(*api.JobInfo)
                rv := r.(*api.JobInfo)

                if lv.Preemptable == rv.Preemptable </span><span class="cov0" title="0">{
                        return 0
                }</span>

                <span class="cov0" title="0">if !lv.Preemptable </span><span class="cov0" title="0">{
                        return -1
                }</span>

                <span class="cov0" title="0">return 1</span>
        }

        <span class="cov8" title="1">jobPipelinedFn := func(obj interface{}) int </span><span class="cov0" title="0">{
                jobInfo := obj.(*api.JobInfo)
                occupied := jobInfo.WaitingTaskNum() + jobInfo.ReadyTaskNum()
                if occupied &gt;= jobInfo.MinAvailable </span><span class="cov0" title="0">{
                        return tutil.Permit
                }</span>
                <span class="cov0" title="0">return tutil.Reject</span>
        }

        <span class="cov8" title="1">jobStarvingFn := func(obj interface{}) bool </span><span class="cov0" title="0">{
                jobInfo := obj.(*api.JobInfo)
                // allow none preemptable elastic job (deployment) preempt task
                if jobInfo.Preemptable </span><span class="cov0" title="0">{
                        return false
                }</span>
                <span class="cov0" title="0">return len(jobInfo.TaskStatusIndex[api.Pending]) &gt; 0</span>
        }

        <span class="cov8" title="1">ssn.AddPredicateFn(tp.Name(), predicateFn)
        ssn.AddNodeOrderFn(tp.Name(), nodeOrderFn)
        ssn.AddPreemptableFn(tp.Name(), preemptableFn)
        ssn.AddVictimTasksFns(tp.Name(), victimsFn)
        ssn.AddJobOrderFn(tp.Name(), jobOrderFn)
        ssn.AddJobPipelinedFn(tp.Name(), jobPipelinedFn)
        ssn.AddJobStarvingFns(tp.Name(), jobStarvingFn)</span>
}

func (tp *tdmPlugin) maxVictims(job *api.JobInfo, victims []*api.TaskInfo) []*api.TaskInfo <span class="cov8" title="1">{
        maxPodEvictNum := tp.getMaxPodEvictNum(job)
        targetNum := util.GetMinInt(maxPodEvictNum, len(victims))
        klog.V(3).Infof("Job &lt;%s/%s&gt; max evict:%v, potential victims number:%v, max victims number:%v",
                job.Namespace, job.Name, maxPodEvictNum, len(victims), targetNum)

        return victims[:targetNum]
}</span>

// get max pod evict number from job budget configure
func (tp *tdmPlugin) getMaxPodEvictNum(job *api.JobInfo) int <span class="cov8" title="1">{
        jobRunningTaskNum := len(job.TaskStatusIndex[api.Running])
        if job.Budget.MaxUnavilable != "" </span><span class="cov8" title="1">{
                maxUnavilable := tp.parseIntStr(job.Budget.MaxUnavilable, len(job.Tasks))
                finalTaskNum := len(job.TaskStatusIndex[api.Succeeded]) + len(job.TaskStatusIndex[api.Failed])
                realUnavilable := len(job.Tasks) - finalTaskNum - jobRunningTaskNum
                if realUnavilable &gt;= maxUnavilable </span><span class="cov0" title="0">{
                        return 0
                }</span>
                <span class="cov8" title="1">return maxUnavilable - realUnavilable</span>
        }

        <span class="cov8" title="1">if job.Budget.MinAvailable != "" </span><span class="cov8" title="1">{
                minAvailable := tp.parseIntStr(job.Budget.MinAvailable, len(job.Tasks))
                if jobRunningTaskNum &gt;= minAvailable </span><span class="cov8" title="1">{
                        return jobRunningTaskNum - minAvailable
                }</span>
        }

        <span class="cov0" title="0">return defaultPodEvictNum</span>
}

func (tp *tdmPlugin) parseIntStr(input string, taskNum int) int <span class="cov8" title="1">{
        resultValue := 0
        tmp := intstr.Parse(input)
        switch tmp.Type </span>{
        case intstr.Int:<span class="cov8" title="1">
                resultValue = tmp.IntValue()</span>
        case intstr.String:<span class="cov8" title="1">
                if v, err := intstr.GetValueFromIntOrPercent(&amp;tmp, taskNum, true); err == nil </span><span class="cov8" title="1">{
                        resultValue = v
                }</span> else<span class="cov0" title="0"> {
                        klog.Warningf("TDM get percent value err: %v", err)
                }</span>
        }

        <span class="cov8" title="1">return resultValue</span>
}

func (tp *tdmPlugin) revocableNodePreemptableTask(rz string, ssn *framework.Session) map[api.JobID][]*api.TaskInfo <span class="cov8" title="1">{
        tasksMap := make(map[api.JobID][]*api.TaskInfo)
        for _, node := range ssn.RevocableNodes </span><span class="cov8" title="1">{
                if node.RevocableZone != rz </span><span class="cov0" title="0">{
                        continue</span>
                }

                <span class="cov8" title="1">for _, task := range node.Tasks </span><span class="cov8" title="1">{
                        if task.Preemptable </span><span class="cov8" title="1">{
                                if task.Status == api.Running </span><span class="cov8" title="1">{
                                        tasksMap[task.Job] = append(tasksMap[task.Job], task)
                                }</span>
                        }
                }
        }

        <span class="cov8" title="1">return tasksMap</span>
}

func (tp *tdmPlugin) OnSessionClose(ssn *framework.Session) {<span class="cov8" title="1">}</span>
</pre>
		
		<pre class="file" id="file82" style="display: none">/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
        "fmt"
        "path/filepath"
        "sync"
        "time"

        "github.com/fsnotify/fsnotify"
        "k8s.io/apimachinery/pkg/util/wait"
        "k8s.io/client-go/rest"
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/filewatcher"
        schedcache "volcano.sh/volcano/pkg/scheduler/cache"
        "volcano.sh/volcano/pkg/scheduler/conf"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/metrics"
)

// Scheduler watches for new unscheduled pods for volcano. It attempts to find
// nodes that they fit on and writes bindings back to the api server.
type Scheduler struct {
        cache          schedcache.Cache
        schedulerConf  string
        fileWatcher    filewatcher.FileWatcher
        schedulePeriod time.Duration
        once           sync.Once

        mutex          sync.Mutex
        actions        []framework.Action
        plugins        []conf.Tier
        configurations []conf.Configuration
}

// NewScheduler returns a scheduler
func NewScheduler(
        config *rest.Config,
        schedulerName string,
        schedulerConf string,
        period time.Duration,
        defaultQueue string,
) (*Scheduler, error) <span class="cov0" title="0">{
        var watcher filewatcher.FileWatcher
        if schedulerConf != "" </span><span class="cov0" title="0">{
                var err error
                path := filepath.Dir(schedulerConf)
                watcher, err = filewatcher.NewFileWatcher(path)
                if err != nil </span><span class="cov0" title="0">{
                        return nil, fmt.Errorf("failed creating filewatcher for %s: %v", schedulerConf, err)
                }</span>
        }

        <span class="cov0" title="0">scheduler := &amp;Scheduler{
                schedulerConf:  schedulerConf,
                fileWatcher:    watcher,
                cache:          schedcache.New(config, schedulerName, defaultQueue),
                schedulePeriod: period,
        }

        return scheduler, nil</span>
}

// Run runs the Scheduler
func (pc *Scheduler) Run(stopCh &lt;-chan struct{}) <span class="cov0" title="0">{
        pc.loadSchedulerConf()
        go pc.watchSchedulerConf(stopCh)
        // Start cache for policy.
        go pc.cache.Run(stopCh)
        pc.cache.WaitForCacheSync(stopCh)
        go wait.Until(pc.runOnce, pc.schedulePeriod, stopCh)
}</span>

func (pc *Scheduler) runOnce() <span class="cov0" title="0">{
        klog.V(4).Infof("Start scheduling ...")
        scheduleStartTime := time.Now()
        defer klog.V(4).Infof("End scheduling ...")

        pc.mutex.Lock()
        actions := pc.actions
        plugins := pc.plugins
        configurations := pc.configurations
        pc.mutex.Unlock()

        ssn := framework.OpenSession(pc.cache, plugins, configurations)
        defer framework.CloseSession(ssn)

        for _, action := range actions </span><span class="cov0" title="0">{
                actionStartTime := time.Now()
                action.Execute(ssn)
                metrics.UpdateActionDuration(action.Name(), metrics.Duration(actionStartTime))
        }</span>
        <span class="cov0" title="0">metrics.UpdateE2eDuration(metrics.Duration(scheduleStartTime))</span>
}

func (pc *Scheduler) loadSchedulerConf() <span class="cov0" title="0">{
        var err error
        pc.once.Do(func() </span><span class="cov0" title="0">{
                pc.actions, pc.plugins, pc.configurations, err = unmarshalSchedulerConf(defaultSchedulerConf)
                if err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("unmarshal scheduler config %s failed: %v", defaultSchedulerConf, err)
                        panic("invalid default configuration")</span>
                }
        })

        <span class="cov0" title="0">var config string
        if len(pc.schedulerConf) != 0 </span><span class="cov0" title="0">{
                if config, err = readSchedulerConf(pc.schedulerConf); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Failed to read scheduler configuration '%s', using previous configuration: %v",
                                pc.schedulerConf, err)
                        return
                }</span>
        }

        <span class="cov0" title="0">actions, plugins, configurations, err := unmarshalSchedulerConf(config)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("scheduler config %s is invalid: %v", config, err)
                return
        }</span>

        <span class="cov0" title="0">pc.mutex.Lock()
        // If it is valid, use the new configuration
        pc.actions = actions
        pc.plugins = plugins
        pc.configurations = configurations
        pc.mutex.Unlock()</span>
}

func (pc *Scheduler) watchSchedulerConf(stopCh &lt;-chan struct{}) <span class="cov0" title="0">{
        if pc.fileWatcher == nil </span><span class="cov0" title="0">{
                return
        }</span>
        <span class="cov0" title="0">eventCh := pc.fileWatcher.Events()
        errCh := pc.fileWatcher.Errors()
        for </span><span class="cov0" title="0">{
                select </span>{
                case event, ok := &lt;-eventCh:<span class="cov0" title="0">
                        if !ok </span><span class="cov0" title="0">{
                                return
                        }</span>
                        <span class="cov0" title="0">klog.V(4).Infof("watch %s event: %v", pc.schedulerConf, event)
                        if event.Op&amp;fsnotify.Write == fsnotify.Write || event.Op&amp;fsnotify.Create == fsnotify.Create </span><span class="cov0" title="0">{
                                pc.loadSchedulerConf()
                        }</span>
                case err, ok := &lt;-errCh:<span class="cov0" title="0">
                        if !ok </span><span class="cov0" title="0">{
                                return
                        }</span>
                        <span class="cov0" title="0">klog.Infof("watch %s error: %v", pc.schedulerConf, err)</span>
                case &lt;-stopCh:<span class="cov0" title="0">
                        return</span>
                }
        }
}
</pre>
		
		<pre class="file" id="file83" style="display: none">/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
        "fmt"
        "io/ioutil"
        "strings"

        "gopkg.in/yaml.v2"

        "volcano.sh/volcano/pkg/scheduler/conf"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/plugins"
)

var defaultSchedulerConf = `
actions: "enqueue, allocate, backfill"
tiers:
- plugins:
  - name: priority
  - name: gang
  - name: conformance
- plugins:
  - name: overcommit
  - name: drf
  - name: predicates
  - name: proportion
  - name: nodeorder
`

func unmarshalSchedulerConf(confStr string) ([]framework.Action, []conf.Tier, []conf.Configuration, error) <span class="cov8" title="1">{
        var actions []framework.Action

        schedulerConf := &amp;conf.SchedulerConfiguration{}

        if err := yaml.Unmarshal([]byte(confStr), schedulerConf); err != nil </span><span class="cov0" title="0">{
                return nil, nil, nil, err
        }</span>
        // Set default settings for each plugin if not set
        <span class="cov8" title="1">for i, tier := range schedulerConf.Tiers </span><span class="cov8" title="1">{
                // drf with hierarchy enabled
                hdrf := false
                // proportion enabled
                proportion := false
                for j := range tier.Plugins </span><span class="cov8" title="1">{
                        if tier.Plugins[j].Name == "drf" &amp;&amp;
                                tier.Plugins[j].EnabledHierarchy != nil &amp;&amp;
                                *tier.Plugins[j].EnabledHierarchy </span><span class="cov0" title="0">{
                                hdrf = true
                        }</span>
                        <span class="cov8" title="1">if tier.Plugins[j].Name == "proportion" </span><span class="cov8" title="1">{
                                proportion = true
                        }</span>
                        <span class="cov8" title="1">plugins.ApplyPluginConfDefaults(&amp;schedulerConf.Tiers[i].Plugins[j])</span>
                }
                <span class="cov8" title="1">if hdrf &amp;&amp; proportion </span><span class="cov0" title="0">{
                        return nil, nil, nil, fmt.Errorf("proportion and drf with hierarchy enabled conflicts")
                }</span>
        }

        <span class="cov8" title="1">actionNames := strings.Split(schedulerConf.Actions, ",")
        for _, actionName := range actionNames </span><span class="cov8" title="1">{
                if action, found := framework.GetAction(strings.TrimSpace(actionName)); found </span><span class="cov8" title="1">{
                        actions = append(actions, action)
                }</span> else<span class="cov0" title="0"> {
                        return nil, nil, nil, fmt.Errorf("failed to found Action %s, ignore it", actionName)
                }</span>
        }

        <span class="cov8" title="1">return actions, schedulerConf.Tiers, schedulerConf.Configurations, nil</span>
}

func readSchedulerConf(confPath string) (string, error) <span class="cov0" title="0">{
        dat, err := ioutil.ReadFile(confPath)
        if err != nil </span><span class="cov0" title="0">{
                return "", err
        }</span>
        <span class="cov0" title="0">return string(dat), nil</span>
}
</pre>
		
		<pre class="file" id="file84" style="display: none">/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
        "container/heap"

        "volcano.sh/volcano/pkg/scheduler/api"
)

//PriorityQueue implements a scheduling queue.
type PriorityQueue struct {
        queue priorityQueue
}

type priorityQueue struct {
        items  []interface{}
        lessFn api.LessFn
}

// NewPriorityQueue returns a PriorityQueue
func NewPriorityQueue(lessFn api.LessFn) *PriorityQueue <span class="cov0" title="0">{
        return &amp;PriorityQueue{
                queue: priorityQueue{
                        items:  make([]interface{}, 0),
                        lessFn: lessFn,
                },
        }
}</span>

// Push pushes element in the priority Queue
func (q *PriorityQueue) Push(it interface{}) <span class="cov0" title="0">{
        heap.Push(&amp;q.queue, it)
}</span>

// Pop pops element in the priority Queue
func (q *PriorityQueue) Pop() interface{} <span class="cov0" title="0">{
        if q.Len() == 0 </span><span class="cov0" title="0">{
                return nil
        }</span>

        <span class="cov0" title="0">return heap.Pop(&amp;q.queue)</span>
}

// Empty check if queue is empty
func (q *PriorityQueue) Empty() bool <span class="cov0" title="0">{
        return q.queue.Len() == 0
}</span>

// Len returns Len of the priority queue
func (q *PriorityQueue) Len() int <span class="cov0" title="0">{
        return q.queue.Len()
}</span>

func (pq *priorityQueue) Len() int <span class="cov0" title="0">{ return len(pq.items) }</span>

func (pq *priorityQueue) Less(i, j int) bool <span class="cov0" title="0">{
        if pq.lessFn == nil </span><span class="cov0" title="0">{
                return i &lt; j
        }</span>

        // We want Pop to give us the highest, not lowest, priority so we use greater than here.
        <span class="cov0" title="0">return pq.lessFn(pq.items[i], pq.items[j])</span>
}

func (pq priorityQueue) Swap(i, j int) <span class="cov0" title="0">{
        pq.items[i], pq.items[j] = pq.items[j], pq.items[i]
}</span>

func (pq *priorityQueue) Push(x interface{}) <span class="cov0" title="0">{
        (*pq).items = append((*pq).items, x)
}</span>

func (pq *priorityQueue) Pop() interface{} <span class="cov0" title="0">{
        old := (*pq).items
        n := len(old)
        item := old[n-1]
        (*pq).items = old[0 : n-1]
        return item
}</span>
</pre>
		
		<pre class="file" id="file85" style="display: none">/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
        "context"
        "fmt"
        "math"
        "math/rand"
        "sort"
        "sync"
        "sync/atomic"

        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"
        k8sframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"

        "volcano.sh/volcano/cmd/scheduler/app/options"
        "volcano.sh/volcano/pkg/scheduler/api"
)

const baselinePercentageOfNodesToFind = 50

var lastProcessedNodeIndex int

// Reservation is used to record target job and locked nodes
var Reservation *ResourceReservation

func init() <span class="cov8" title="1">{
        Reservation = NewResourceReservation()
}</span>

// CalculateNumOfFeasibleNodesToFind returns the number of feasible nodes that once found,
// the scheduler stops its search for more feasible nodes.
func CalculateNumOfFeasibleNodesToFind(numAllNodes int32) (numNodes int32) <span class="cov0" title="0">{
        opts := options.ServerOpts
        if numAllNodes &lt;= opts.MinNodesToFind || opts.PercentageOfNodesToFind &gt;= 100 </span><span class="cov0" title="0">{
                return numAllNodes
        }</span>

        <span class="cov0" title="0">adaptivePercentage := opts.PercentageOfNodesToFind
        if adaptivePercentage &lt;= 0 </span><span class="cov0" title="0">{
                adaptivePercentage = baselinePercentageOfNodesToFind - numAllNodes/125
                if adaptivePercentage &lt; opts.MinPercentageOfNodesToFind </span><span class="cov0" title="0">{
                        adaptivePercentage = opts.MinPercentageOfNodesToFind
                }</span>
        }

        <span class="cov0" title="0">numNodes = numAllNodes * adaptivePercentage / 100
        if numNodes &lt; opts.MinNodesToFind </span><span class="cov0" title="0">{
                numNodes = opts.MinNodesToFind
        }</span>
        <span class="cov0" title="0">return numNodes</span>
}

// PredicateNodes returns the specified number of nodes that fit a task
func PredicateNodes(task *api.TaskInfo, nodes []*api.NodeInfo, fn api.PredicateFn) ([]*api.NodeInfo, *api.FitErrors) <span class="cov0" title="0">{
        //var workerLock sync.Mutex

        var errorLock sync.Mutex
        fe := api.NewFitErrors()

        allNodes := len(nodes)
        if allNodes == 0 </span><span class="cov0" title="0">{
                return make([]*api.NodeInfo, 0), fe
        }</span>
        <span class="cov0" title="0">numNodesToFind := CalculateNumOfFeasibleNodesToFind(int32(allNodes))

        //allocate enough space to avoid growing it
        predicateNodes := make([]*api.NodeInfo, numNodesToFind)

        numFoundNodes := int32(0)
        processedNodes := int32(0)

        //create a context with cancellation
        ctx, cancel := context.WithCancel(context.Background())

        checkNode := func(index int) </span><span class="cov0" title="0">{
                // Check the nodes starting from where is left off in the previous scheduling cycle,
                // to make sure all nodes have the same chance of being examined across pods.
                node := nodes[(lastProcessedNodeIndex+index)%allNodes]
                atomic.AddInt32(&amp;processedNodes, 1)
                klog.V(4).Infof("Considering Task &lt;%v/%v&gt; on node &lt;%v&gt;: &lt;%v&gt; vs. &lt;%v&gt;",
                        task.Namespace, task.Name, node.Name, task.Resreq, node.Idle)

                // TODO (k82cn): Enable eCache for performance improvement.
                if err := fn(task, node); err != nil </span><span class="cov0" title="0">{
                        klog.V(3).Infof("Predicates failed for task &lt;%s/%s&gt; on node &lt;%s&gt;: %v",
                                task.Namespace, task.Name, node.Name, err)
                        errorLock.Lock()
                        fe.SetNodeError(node.Name, err)
                        errorLock.Unlock()
                        return
                }</span>

                //check if the number of found nodes is more than the numNodesTofind
                <span class="cov0" title="0">length := atomic.AddInt32(&amp;numFoundNodes, 1)
                if length &gt; numNodesToFind </span><span class="cov0" title="0">{
                        cancel()
                        atomic.AddInt32(&amp;numFoundNodes, -1)
                }</span> else<span class="cov0" title="0"> {
                        predicateNodes[length-1] = node
                }</span>
        }

        //workqueue.ParallelizeUntil(context.TODO(), 16, len(nodes), checkNode)
        <span class="cov0" title="0">workqueue.ParallelizeUntil(ctx, 16, allNodes, checkNode)

        //processedNodes := int(numFoundNodes) + len(filteredNodesStatuses) + len(failedPredicateMap)
        lastProcessedNodeIndex = (lastProcessedNodeIndex + int(processedNodes)) % allNodes
        predicateNodes = predicateNodes[:numFoundNodes]
        return predicateNodes, fe</span>
}

// PrioritizeNodes returns a map whose key is node's score and value are corresponding nodes
func PrioritizeNodes(task *api.TaskInfo, nodes []*api.NodeInfo, batchFn api.BatchNodeOrderFn, mapFn api.NodeOrderMapFn, reduceFn api.NodeOrderReduceFn) map[float64][]*api.NodeInfo <span class="cov0" title="0">{
        pluginNodeScoreMap := map[string]k8sframework.NodeScoreList{}
        nodeOrderScoreMap := map[string]float64{}
        nodeScores := map[float64][]*api.NodeInfo{}
        var workerLock sync.Mutex
        scoreNode := func(index int) </span><span class="cov0" title="0">{
                node := nodes[index]
                mapScores, orderScore, err := mapFn(task, node)
                if err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("Error in Calculating Priority for the node:%v", err)
                        return
                }</span>

                <span class="cov0" title="0">workerLock.Lock()
                for plugin, score := range mapScores </span><span class="cov0" title="0">{
                        nodeScoreMap, ok := pluginNodeScoreMap[plugin]
                        if !ok </span><span class="cov0" title="0">{
                                nodeScoreMap = k8sframework.NodeScoreList{}
                        }</span>
                        <span class="cov0" title="0">hp := k8sframework.NodeScore{}
                        hp.Name = node.Name
                        hp.Score = int64(math.Floor(score))
                        pluginNodeScoreMap[plugin] = append(nodeScoreMap, hp)</span>
                }
                <span class="cov0" title="0">nodeOrderScoreMap[node.Name] = orderScore
                workerLock.Unlock()</span>
        }
        <span class="cov0" title="0">workqueue.ParallelizeUntil(context.TODO(), 16, len(nodes), scoreNode)
        reduceScores, err := reduceFn(task, pluginNodeScoreMap)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Error in Calculating Priority for the node:%v", err)
                return nodeScores
        }</span>

        <span class="cov0" title="0">batchNodeScore, err := batchFn(task, nodes)
        if err != nil </span><span class="cov0" title="0">{
                klog.Errorf("Error in Calculating batch Priority for the node, err %v", err)
                return nodeScores
        }</span>

        <span class="cov0" title="0">for _, node := range nodes </span><span class="cov0" title="0">{
                if score, found := reduceScores[node.Name]; found </span><span class="cov0" title="0">{
                        if orderScore, ok := nodeOrderScoreMap[node.Name]; ok </span><span class="cov0" title="0">{
                                score += orderScore
                        }</span>
                        <span class="cov0" title="0">if batchScore, ok := batchNodeScore[node.Name]; ok </span><span class="cov0" title="0">{
                                score += batchScore
                        }</span>
                        <span class="cov0" title="0">nodeScores[score] = append(nodeScores[score], node)</span>
                } else<span class="cov0" title="0"> {
                        // If no plugin is applied to this node, the default is 0.0
                        score = 0.0
                        if orderScore, ok := nodeOrderScoreMap[node.Name]; ok </span><span class="cov0" title="0">{
                                score += orderScore
                        }</span>
                        <span class="cov0" title="0">if batchScore, ok := batchNodeScore[node.Name]; ok </span><span class="cov0" title="0">{
                                score += batchScore
                        }</span>
                        <span class="cov0" title="0">nodeScores[score] = append(nodeScores[score], node)</span>
                }
        }
        <span class="cov0" title="0">return nodeScores</span>
}

// SortNodes returns nodes by order of score
func SortNodes(nodeScores map[float64][]*api.NodeInfo) []*api.NodeInfo <span class="cov0" title="0">{
        var nodesInorder []*api.NodeInfo
        var keys []float64
        for key := range nodeScores </span><span class="cov0" title="0">{
                keys = append(keys, key)
        }</span>
        <span class="cov0" title="0">sort.Sort(sort.Reverse(sort.Float64Slice(keys)))
        for _, key := range keys </span><span class="cov0" title="0">{
                nodes := nodeScores[key]
                nodesInorder = append(nodesInorder, nodes...)
        }</span>
        <span class="cov0" title="0">return nodesInorder</span>
}

// SelectBestNode returns best node whose score is highest, pick one randomly if there are many nodes with same score.
func SelectBestNode(nodeScores map[float64][]*api.NodeInfo) *api.NodeInfo <span class="cov8" title="1">{
        var bestNodes []*api.NodeInfo
        maxScore := -1.0
        for score, nodes := range nodeScores </span><span class="cov8" title="1">{
                if score &gt; maxScore </span><span class="cov8" title="1">{
                        maxScore = score
                        bestNodes = nodes
                }</span>
        }

        <span class="cov8" title="1">if len(bestNodes) == 0 </span><span class="cov8" title="1">{
                return nil
        }</span>

        <span class="cov8" title="1">return bestNodes[rand.Intn(len(bestNodes))]</span>
}

// GetNodeList returns values of the map 'nodes'
func GetNodeList(nodes map[string]*api.NodeInfo, nodeList []string) []*api.NodeInfo <span class="cov0" title="0">{
        result := make([]*api.NodeInfo, 0, len(nodeList))
        for _, nodename := range nodeList </span><span class="cov0" title="0">{
                if ni, ok := nodes[nodename]; ok </span><span class="cov0" title="0">{
                        result = append(result, ni)
                }</span>
        }
        <span class="cov0" title="0">return result</span>
}

// ValidateVictims returns an error if the resources of the victims can't satisfy the preemptor
func ValidateVictims(preemptor *api.TaskInfo, node *api.NodeInfo, victims []*api.TaskInfo) error <span class="cov0" title="0">{
        if len(victims) == 0 </span><span class="cov0" title="0">{
                return fmt.Errorf("no victims")
        }</span>
        <span class="cov0" title="0">futureIdle := node.FutureIdle()
        for _, victim := range victims </span><span class="cov0" title="0">{
                futureIdle.Add(victim.Resreq)
        }</span>
        // Every resource of the preemptor needs to be less or equal than corresponding
        // idle resource after preemption.
        <span class="cov0" title="0">if !preemptor.InitResreq.LessEqual(futureIdle, api.Zero) </span><span class="cov0" title="0">{
                return fmt.Errorf("not enough resources: requested &lt;%v&gt;, but future idle &lt;%v&gt;",
                        preemptor.InitResreq, futureIdle)
        }</span>
        <span class="cov0" title="0">return nil</span>
}

// ResourceReservation is struct used for resource reservation
type ResourceReservation struct {
        TargetJob   *api.JobInfo
        LockedNodes map[string]*api.NodeInfo
}

// NewResourceReservation is used to create global instance
func NewResourceReservation() *ResourceReservation <span class="cov8" title="1">{
        return &amp;ResourceReservation{
                TargetJob:   nil,
                LockedNodes: map[string]*api.NodeInfo{},
        }
}</span>

// GetMinInt return minimum int from vals
func GetMinInt(vals ...int) int <span class="cov8" title="1">{
        if len(vals) == 0 </span><span class="cov8" title="1">{
                return 0
        }</span>

        <span class="cov8" title="1">min := vals[0]
        for _, val := range vals </span><span class="cov8" title="1">{
                if val &lt;= min </span><span class="cov8" title="1">{
                        min = val
                }</span>
        }
        <span class="cov8" title="1">return min</span>
}
</pre>
		
		<pre class="file" id="file86" style="display: none">/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
        "fmt"
        "sync"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/resource"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
        volumescheduling "k8s.io/kubernetes/pkg/controller/volume/scheduling"

        schedulingv2 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/scheduler/api"
)

// BuildResourceList builts resource list object
func BuildResourceList(cpu string, memory string) v1.ResourceList <span class="cov0" title="0">{
        return v1.ResourceList{
                v1.ResourceCPU:      resource.MustParse(cpu),
                v1.ResourceMemory:   resource.MustParse(memory),
                api.GPUResourceName: resource.MustParse("0"),
        }
}</span>

// BuildResourceListWithGPU builts resource list with GPU
func BuildResourceListWithGPU(cpu string, memory string, GPU string) v1.ResourceList <span class="cov0" title="0">{
        return v1.ResourceList{
                v1.ResourceCPU:      resource.MustParse(cpu),
                v1.ResourceMemory:   resource.MustParse(memory),
                api.GPUResourceName: resource.MustParse(GPU),
        }
}</span>

// BuildNode builts node object
func BuildNode(name string, alloc v1.ResourceList, labels map[string]string) *v1.Node <span class="cov0" title="0">{
        return &amp;v1.Node{
                ObjectMeta: metav1.ObjectMeta{
                        Name:        name,
                        Labels:      labels,
                        Annotations: map[string]string{},
                },
                Status: v1.NodeStatus{
                        Capacity:    alloc,
                        Allocatable: alloc,
                },
        }
}</span>

// BuildPod builts Pod object
func BuildPod(namespace, name, nodename string, p v1.PodPhase, req v1.ResourceList, groupName string, labels map[string]string, selector map[string]string) *v1.Pod <span class="cov0" title="0">{
        return &amp;v1.Pod{
                ObjectMeta: metav1.ObjectMeta{
                        UID:       types.UID(fmt.Sprintf("%v-%v", namespace, name)),
                        Name:      name,
                        Namespace: namespace,
                        Labels:    labels,
                        Annotations: map[string]string{
                                schedulingv2.KubeGroupNameAnnotationKey: groupName,
                        },
                },
                Status: v1.PodStatus{
                        Phase: p,
                },
                Spec: v1.PodSpec{
                        NodeName:     nodename,
                        NodeSelector: selector,
                        Containers: []v1.Container{
                                {
                                        Resources: v1.ResourceRequirements{
                                                Requests: req,
                                        },
                                },
                        },
                },
        }
}</span>

// FakeBinder is used as fake binder
type FakeBinder struct {
        sync.Mutex
        Binds   map[string]string
        Channel chan string
}

// Bind used by fake binder struct to bind pods
func (fb *FakeBinder) Bind(p *v1.Pod, hostname string) error <span class="cov0" title="0">{
        fb.Lock()
        defer fb.Unlock()

        key := fmt.Sprintf("%v/%v", p.Namespace, p.Name)
        fb.Binds[key] = hostname

        fb.Channel &lt;- key

        return nil
}</span>

// FakeEvictor is used as fake evictor
type FakeEvictor struct {
        sync.Mutex
        evicts  []string
        Channel chan string
}

// Evicts returns copy of evicted pods.
func (fe *FakeEvictor) Evicts() []string <span class="cov0" title="0">{
        fe.Lock()
        defer fe.Unlock()
        return append([]string{}, fe.evicts...)
}</span>

// Evict is used by fake evictor to evict pods
func (fe *FakeEvictor) Evict(p *v1.Pod, reason string) error <span class="cov0" title="0">{
        fe.Lock()
        defer fe.Unlock()

        fmt.Println("PodName: ", p.Name)
        key := fmt.Sprintf("%v/%v", p.Namespace, p.Name)
        fe.evicts = append(fe.evicts, key)

        fe.Channel &lt;- key

        return nil
}</span>

// FakeStatusUpdater is used for fake status update
type FakeStatusUpdater struct {
}

// UpdatePodCondition is a empty function
func (ftsu *FakeStatusUpdater) UpdatePodCondition(pod *v1.Pod, podCondition *v1.PodCondition) (*v1.Pod, error) <span class="cov0" title="0">{
        // do nothing here
        return nil, nil
}</span>

// UpdatePodGroup is a empty function
func (ftsu *FakeStatusUpdater) UpdatePodGroup(pg *api.PodGroup) (*api.PodGroup, error) <span class="cov0" title="0">{
        // do nothing here
        return nil, nil
}</span>

// FakeVolumeBinder is used as fake volume binder
type FakeVolumeBinder struct {
}

// AllocateVolumes is a empty function
func (fvb *FakeVolumeBinder) AllocateVolumes(task *api.TaskInfo, hostname string, podVolumes *volumescheduling.PodVolumes) error <span class="cov0" title="0">{
        return nil
}</span>

// BindVolumes is a empty function
func (fvb *FakeVolumeBinder) BindVolumes(task *api.TaskInfo, podVolumes *volumescheduling.PodVolumes) error <span class="cov0" title="0">{
        return nil
}</span>

// GetPodVolumes is a empty function
func (fvb *FakeVolumeBinder) GetPodVolumes(task *api.TaskInfo, node *v1.Node) (*volumescheduling.PodVolumes, error) <span class="cov0" title="0">{
        return nil, nil
}</span>
</pre>
		
		<pre class="file" id="file87" style="display: none">/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mutate

import (
        "encoding/json"
        "fmt"
        "strconv"

        "k8s.io/api/admission/v1beta1"
        whv1beta1 "k8s.io/api/admissionregistration/v1beta1"
        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/volcano/pkg/webhooks/router"
        "volcano.sh/volcano/pkg/webhooks/schema"
        "volcano.sh/volcano/pkg/webhooks/util"
)

const (
        // DefaultQueue constant stores the name of the queue as "default"
        DefaultQueue = "default"
        // DefaultMaxRetry is the default number of retries.
        DefaultMaxRetry = 3

        defaultSchedulerName = "volcano"

        defaultMaxRetry int32 = 3
)

func init() <span class="cov8" title="1">{
        router.RegisterAdmission(service)
}</span>

var service = &amp;router.AdmissionService{
        Path: "/jobs/mutate",
        Func: Jobs,

        MutatingConfig: &amp;whv1beta1.MutatingWebhookConfiguration{
                Webhooks: []whv1beta1.MutatingWebhook{{
                        Name: "mutatejob.volcano.sh",
                        Rules: []whv1beta1.RuleWithOperations{
                                {
                                        Operations: []whv1beta1.OperationType{whv1beta1.Create},
                                        Rule: whv1beta1.Rule{
                                                APIGroups:   []string{"batch.volcano.sh"},
                                                APIVersions: []string{"v1alpha1"},
                                                Resources:   []string{"jobs"},
                                        },
                                },
                        },
                }},
        },
}

type patchOperation struct {
        Op    string      `json:"op"`
        Path  string      `json:"path"`
        Value interface{} `json:"value,omitempty"`
}

// Jobs mutate jobs.
func Jobs(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse <span class="cov0" title="0">{
        klog.V(3).Infof("mutating jobs")

        job, err := schema.DecodeJob(ar.Request.Object, ar.Request.Resource)
        if err != nil </span><span class="cov0" title="0">{
                return util.ToAdmissionResponse(err)
        }</span>

        <span class="cov0" title="0">var patchBytes []byte
        switch ar.Request.Operation </span>{
        case v1beta1.Create:<span class="cov0" title="0">
                patchBytes, _ = createPatch(job)</span>
        default:<span class="cov0" title="0">
                err = fmt.Errorf("expect operation to be 'CREATE' ")
                return util.ToAdmissionResponse(err)</span>
        }

        <span class="cov0" title="0">klog.V(3).Infof("AdmissionResponse: patch=%v", string(patchBytes))
        reviewResponse := v1beta1.AdmissionResponse{
                Allowed: true,
                Patch:   patchBytes,
        }
        pt := v1beta1.PatchTypeJSONPatch
        reviewResponse.PatchType = &amp;pt

        return &amp;reviewResponse</span>
}

func createPatch(job *v1alpha1.Job) ([]byte, error) <span class="cov0" title="0">{
        var patch []patchOperation
        pathQueue := patchDefaultQueue(job)
        if pathQueue != nil </span><span class="cov0" title="0">{
                patch = append(patch, *pathQueue)
        }</span>
        <span class="cov0" title="0">pathScheduler := patchDefaultScheduler(job)
        if pathScheduler != nil </span><span class="cov0" title="0">{
                patch = append(patch, *pathScheduler)
        }</span>
        <span class="cov0" title="0">pathMaxRetry := patchDefaultMaxRetry(job)
        if pathMaxRetry != nil </span><span class="cov0" title="0">{
                patch = append(patch, *pathMaxRetry)
        }</span>
        <span class="cov0" title="0">pathSpec := mutateSpec(job.Spec.Tasks, "/spec/tasks")
        if pathSpec != nil </span><span class="cov0" title="0">{
                patch = append(patch, *pathSpec)
        }</span>
        <span class="cov0" title="0">pathMinAvailable := patchDefaultMinAvailable(job)
        if pathMinAvailable != nil </span><span class="cov0" title="0">{
                patch = append(patch, *pathMinAvailable)
        }</span>
        <span class="cov0" title="0">return json.Marshal(patch)</span>
}

func patchDefaultQueue(job *v1alpha1.Job) *patchOperation <span class="cov0" title="0">{
        //Add default queue if not specified.
        if job.Spec.Queue == "" </span><span class="cov0" title="0">{
                return &amp;patchOperation{Op: "add", Path: "/spec/queue", Value: DefaultQueue}
        }</span>
        <span class="cov0" title="0">return nil</span>
}

func patchDefaultScheduler(job *v1alpha1.Job) *patchOperation <span class="cov0" title="0">{
        // Add default scheduler name if not specified.
        if job.Spec.SchedulerName == "" </span><span class="cov0" title="0">{
                return &amp;patchOperation{Op: "add", Path: "/spec/schedulerName", Value: defaultSchedulerName}
        }</span>
        <span class="cov0" title="0">return nil</span>
}

func patchDefaultMaxRetry(job *v1alpha1.Job) *patchOperation <span class="cov0" title="0">{
        // Add default maxRetry if maxRetry is zero.
        if job.Spec.MaxRetry == 0 </span><span class="cov0" title="0">{
                return &amp;patchOperation{Op: "add", Path: "/spec/maxRetry", Value: DefaultMaxRetry}
        }</span>
        <span class="cov0" title="0">return nil</span>
}

func patchDefaultMinAvailable(job *v1alpha1.Job) *patchOperation <span class="cov0" title="0">{
        // Add default minAvailable if minAvailable is zero.
        if job.Spec.MinAvailable == 0 </span><span class="cov0" title="0">{
                var jobMinAvailable int32
                for _, task := range job.Spec.Tasks </span><span class="cov0" title="0">{
                        if task.MinAvailable != nil </span><span class="cov0" title="0">{
                                jobMinAvailable += *task.MinAvailable
                        }</span> else<span class="cov0" title="0"> {
                                jobMinAvailable += task.Replicas
                        }</span>
                }

                <span class="cov0" title="0">return &amp;patchOperation{Op: "add", Path: "/spec/minAvailable", Value: jobMinAvailable}</span>
        }
        <span class="cov0" title="0">return nil</span>
}

func mutateSpec(tasks []v1alpha1.TaskSpec, basePath string) *patchOperation <span class="cov8" title="1">{
        patched := false
        for index := range tasks </span><span class="cov8" title="1">{
                // add default task name
                taskName := tasks[index].Name
                if len(taskName) == 0 </span><span class="cov8" title="1">{
                        patched = true
                        tasks[index].Name = v1alpha1.DefaultTaskSpec + strconv.Itoa(index)
                }</span>

                <span class="cov8" title="1">if tasks[index].Template.Spec.HostNetwork &amp;&amp; tasks[index].Template.Spec.DNSPolicy == "" </span><span class="cov0" title="0">{
                        patched = true
                        tasks[index].Template.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
                }</span>

                <span class="cov8" title="1">if tasks[index].MinAvailable == nil </span><span class="cov8" title="1">{
                        patched = true
                        minAvailable := tasks[index].Replicas
                        tasks[index].MinAvailable = &amp;minAvailable
                }</span>

                <span class="cov8" title="1">if tasks[index].MaxRetry == 0 </span><span class="cov8" title="1">{
                        patched = true
                        tasks[index].MaxRetry = defaultMaxRetry
                }</span>
        }
        <span class="cov8" title="1">if !patched </span><span class="cov0" title="0">{
                return nil
        }</span>
        <span class="cov8" title="1">return &amp;patchOperation{
                Op:    "replace",
                Path:  basePath,
                Value: tasks,
        }</span>
}
</pre>
		
		<pre class="file" id="file88" style="display: none">/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validate

import (
        "context"
        "fmt"
        "strings"

        "k8s.io/api/admission/v1beta1"
        whv1beta1 "k8s.io/api/admissionregistration/v1beta1"
        v1 "k8s.io/api/core/v1"
        apiequality "k8s.io/apimachinery/pkg/api/equality"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/util/validation"
        "k8s.io/apimachinery/pkg/util/validation/field"
        "k8s.io/klog"
        k8score "k8s.io/kubernetes/pkg/apis/core"
        k8scorev1 "k8s.io/kubernetes/pkg/apis/core/v1"
        v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
        k8scorevalid "k8s.io/kubernetes/pkg/apis/core/validation"

        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        jobhelpers "volcano.sh/volcano/pkg/controllers/job/helpers"
        "volcano.sh/volcano/pkg/controllers/job/plugins"
        "volcano.sh/volcano/pkg/webhooks/router"
        "volcano.sh/volcano/pkg/webhooks/schema"
        "volcano.sh/volcano/pkg/webhooks/util"
)

func init() <span class="cov8" title="1">{
        router.RegisterAdmission(service)
}</span>

var service = &amp;router.AdmissionService{
        Path: "/jobs/validate",
        Func: AdmitJobs,

        Config: config,

        ValidatingConfig: &amp;whv1beta1.ValidatingWebhookConfiguration{
                Webhooks: []whv1beta1.ValidatingWebhook{{
                        Name: "validatejob.volcano.sh",
                        Rules: []whv1beta1.RuleWithOperations{
                                {
                                        Operations: []whv1beta1.OperationType{whv1beta1.Create, whv1beta1.Update},
                                        Rule: whv1beta1.Rule{
                                                APIGroups:   []string{"batch.volcano.sh"},
                                                APIVersions: []string{"v1alpha1"},
                                                Resources:   []string{"jobs"},
                                        },
                                },
                        },
                }},
        },
}

var config = &amp;router.AdmissionServiceConfig{}

// AdmitJobs is to admit jobs and return response.
func AdmitJobs(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse <span class="cov0" title="0">{
        klog.V(3).Infof("admitting jobs -- %s", ar.Request.Operation)

        job, err := schema.DecodeJob(ar.Request.Object, ar.Request.Resource)
        if err != nil </span><span class="cov0" title="0">{
                return util.ToAdmissionResponse(err)
        }</span>
        <span class="cov0" title="0">var msg string
        reviewResponse := v1beta1.AdmissionResponse{}
        reviewResponse.Allowed = true

        switch ar.Request.Operation </span>{
        case v1beta1.Create:<span class="cov0" title="0">
                msg = validateJobCreate(job, &amp;reviewResponse)</span>
        case v1beta1.Update:<span class="cov0" title="0">
                oldJob, err := schema.DecodeJob(ar.Request.OldObject, ar.Request.Resource)
                if err != nil </span><span class="cov0" title="0">{
                        return util.ToAdmissionResponse(err)
                }</span>
                <span class="cov0" title="0">err = validateJobUpdate(oldJob, job)
                if err != nil </span><span class="cov0" title="0">{
                        return util.ToAdmissionResponse(err)
                }</span>
        default:<span class="cov0" title="0">
                err := fmt.Errorf("expect operation to be 'CREATE' or 'UPDATE'")
                return util.ToAdmissionResponse(err)</span>
        }

        <span class="cov0" title="0">if !reviewResponse.Allowed </span><span class="cov0" title="0">{
                reviewResponse.Result = &amp;metav1.Status{Message: strings.TrimSpace(msg)}
        }</span>
        <span class="cov0" title="0">return &amp;reviewResponse</span>
}

func validateJobCreate(job *v1alpha1.Job, reviewResponse *v1beta1.AdmissionResponse) string <span class="cov8" title="1">{
        var msg string
        taskNames := map[string]string{}
        var totalReplicas int32

        if job.Spec.MinAvailable &lt; 0 </span><span class="cov8" title="1">{
                reviewResponse.Allowed = false
                return "job 'minAvailable' must be &gt;= 0."
        }</span>

        <span class="cov8" title="1">if job.Spec.MaxRetry &lt; 0 </span><span class="cov8" title="1">{
                reviewResponse.Allowed = false
                return "'maxRetry' cannot be less than zero."
        }</span>

        <span class="cov8" title="1">if job.Spec.TTLSecondsAfterFinished != nil &amp;&amp; *job.Spec.TTLSecondsAfterFinished &lt; 0 </span><span class="cov8" title="1">{
                reviewResponse.Allowed = false
                return "'ttlSecondsAfterFinished' cannot be less than zero."
        }</span>

        <span class="cov8" title="1">if len(job.Spec.Tasks) == 0 </span><span class="cov8" title="1">{
                reviewResponse.Allowed = false
                return "No task specified in job spec"
        }</span>

        <span class="cov8" title="1">for index, task := range job.Spec.Tasks </span><span class="cov8" title="1">{
                if task.Replicas &lt; 0 </span><span class="cov8" title="1">{
                        msg += fmt.Sprintf(" 'replicas' &lt; 0 in task: %s;", task.Name)
                }</span>

                <span class="cov8" title="1">if task.MinAvailable != nil &amp;&amp; *task.MinAvailable &gt; task.Replicas </span><span class="cov0" title="0">{
                        msg += fmt.Sprintf(" 'minAvailable' is greater than 'replicas' in task: %s, job: %s", task.Name, job.Name)
                }</span>

                // count replicas
                <span class="cov8" title="1">totalReplicas += task.Replicas

                // validate task name
                if errMsgs := validation.IsDNS1123Label(task.Name); len(errMsgs) &gt; 0 </span><span class="cov8" title="1">{
                        msg += fmt.Sprintf(" %v;", errMsgs)
                }</span>

                // duplicate task name
                <span class="cov8" title="1">if _, found := taskNames[task.Name]; found </span><span class="cov8" title="1">{
                        msg += fmt.Sprintf(" duplicated task name %s;", task.Name)
                        break</span>
                } else<span class="cov8" title="1"> {
                        taskNames[task.Name] = task.Name
                }</span>

                <span class="cov8" title="1">if err := validatePolicies(task.Policies, field.NewPath("spec.tasks.policies")); err != nil </span><span class="cov8" title="1">{
                        msg += err.Error() + fmt.Sprintf(" valid events are %v, valid actions are %v",
                                getValidEvents(), getValidActions())
                }</span>
                <span class="cov8" title="1">podName := jobhelpers.MakePodName(job.Name, task.Name, index)
                msg += validateK8sPodNameLength(podName)
                msg += validateTaskTemplate(task, job, index)</span>
        }

        <span class="cov8" title="1">msg += validateJobName(job)

        if totalReplicas &lt; job.Spec.MinAvailable </span><span class="cov8" title="1">{
                msg += "job 'minAvailable' should not be greater than total replicas in tasks;"
        }</span>

        <span class="cov8" title="1">if err := validatePolicies(job.Spec.Policies, field.NewPath("spec.policies")); err != nil </span><span class="cov8" title="1">{
                msg = msg + err.Error() + fmt.Sprintf(" valid events are %v, valid actions are %v;",
                        getValidEvents(), getValidActions())
        }</span>

        // invalid job plugins
        <span class="cov8" title="1">if len(job.Spec.Plugins) != 0 </span><span class="cov8" title="1">{
                for name := range job.Spec.Plugins </span><span class="cov8" title="1">{
                        if _, found := plugins.GetPluginBuilder(name); !found </span><span class="cov8" title="1">{
                                msg += fmt.Sprintf(" unable to find job plugin: %s", name)
                        }</span>
                }
        }

        <span class="cov8" title="1">if err := validateIO(job.Spec.Volumes); err != nil </span><span class="cov8" title="1">{
                msg += err.Error()
        }</span>

        <span class="cov8" title="1">queue, err := config.VolcanoClient.SchedulingV1beta1().Queues().Get(context.TODO(), job.Spec.Queue, metav1.GetOptions{})
        if err != nil </span><span class="cov8" title="1">{
                msg += fmt.Sprintf(" unable to find job queue: %v", err)
        }</span> else<span class="cov8" title="1"> if queue.Status.State != schedulingv1beta1.QueueStateOpen </span><span class="cov0" title="0">{
                msg += fmt.Sprintf("can only submit job to queue with state `Open`, "+
                        "queue `%s` status is `%s`", queue.Name, queue.Status.State)
        }</span>

        <span class="cov8" title="1">if msg != "" </span><span class="cov8" title="1">{
                reviewResponse.Allowed = false
        }</span>

        <span class="cov8" title="1">return msg</span>
}

func validateJobUpdate(old, new *v1alpha1.Job) error <span class="cov8" title="1">{
        var totalReplicas int32
        for _, task := range new.Spec.Tasks </span><span class="cov8" title="1">{
                if task.Replicas &lt; 0 </span><span class="cov0" title="0">{
                        return fmt.Errorf("'replicas' must be &gt;= 0 in task: %s", task.Name)
                }</span>

                <span class="cov8" title="1">if task.MinAvailable != nil &amp;&amp; *task.MinAvailable &gt; task.Replicas </span><span class="cov0" title="0">{
                        return fmt.Errorf("'minAvailable' must be &lt;= 'replicas' in task: %s;", task.Name)
                }</span>
                // count replicas
                <span class="cov8" title="1">totalReplicas += task.Replicas</span>
        }
        <span class="cov8" title="1">if new.Spec.MinAvailable &gt; totalReplicas </span><span class="cov8" title="1">{
                return fmt.Errorf("job 'minAvailable' must not be greater than total replicas")
        }</span>
        <span class="cov8" title="1">if new.Spec.MinAvailable &lt; 0 </span><span class="cov8" title="1">{
                return fmt.Errorf("job 'minAvailable' must be &gt;= 0")
        }</span>

        <span class="cov8" title="1">if len(old.Spec.Tasks) != len(new.Spec.Tasks) </span><span class="cov8" title="1">{
                return fmt.Errorf("job updates may not add or remove tasks")
        }</span>
        // other fields under spec are not allowed to mutate
        <span class="cov8" title="1">new.Spec.MinAvailable = old.Spec.MinAvailable
        new.Spec.PriorityClassName = old.Spec.PriorityClassName
        for i := range new.Spec.Tasks </span><span class="cov8" title="1">{
                new.Spec.Tasks[i].Replicas = old.Spec.Tasks[i].Replicas
                new.Spec.Tasks[i].MinAvailable = old.Spec.Tasks[i].MinAvailable
        }</span>

        // job controller will update the pvc name if not provided
        <span class="cov8" title="1">for i := range new.Spec.Volumes </span><span class="cov0" title="0">{
                if new.Spec.Volumes[i].VolumeClaim != nil </span><span class="cov0" title="0">{
                        new.Spec.Volumes[i].VolumeClaimName = ""
                }</span>
        }
        <span class="cov8" title="1">for i := range old.Spec.Volumes </span><span class="cov0" title="0">{
                if old.Spec.Volumes[i].VolumeClaim != nil </span><span class="cov0" title="0">{
                        old.Spec.Volumes[i].VolumeClaimName = ""
                }</span>
        }

        <span class="cov8" title="1">if !apiequality.Semantic.DeepEqual(new.Spec, old.Spec) </span><span class="cov8" title="1">{
                return fmt.Errorf("job updates may not change fields other than `minAvailable`, `tasks[*].replicas under spec`")
        }</span>

        <span class="cov8" title="1">return nil</span>
}

func validateTaskTemplate(task v1alpha1.TaskSpec, job *v1alpha1.Job, index int) string <span class="cov8" title="1">{
        var v1PodTemplate v1.PodTemplate
        v1PodTemplate.Template = *task.Template.DeepCopy()
        k8scorev1.SetObjectDefaults_PodTemplate(&amp;v1PodTemplate)

        var coreTemplateSpec k8score.PodTemplateSpec
        k8scorev1.Convert_v1_PodTemplateSpec_To_core_PodTemplateSpec(&amp;v1PodTemplate.Template, &amp;coreTemplateSpec, nil)

        // Skip verify container SecurityContex.Privileged as it depends on
        // the kube-apiserver `allow-privileged` flag.
        for i, container := range coreTemplateSpec.Spec.Containers </span><span class="cov8" title="1">{
                if container.SecurityContext != nil &amp;&amp; container.SecurityContext.Privileged != nil </span><span class="cov8" title="1">{
                        coreTemplateSpec.Spec.Containers[i].SecurityContext.Privileged = nil
                }</span>
        }

        <span class="cov8" title="1">corePodTemplate := k8score.PodTemplate{
                ObjectMeta: metav1.ObjectMeta{
                        Name:      task.Name,
                        Namespace: job.Namespace,
                },
                Template: coreTemplateSpec,
        }

        if allErrs := k8scorevalid.ValidatePodTemplate(&amp;corePodTemplate); len(allErrs) &gt; 0 </span><span class="cov8" title="1">{
                msg := fmt.Sprintf("spec.task[%d].", index)
                for index := range allErrs </span><span class="cov8" title="1">{
                        msg += allErrs[index].Error() + ". "
                }</span>
                <span class="cov8" title="1">return msg</span>
        }

        <span class="cov8" title="1">msg := validateTaskTopoPolicy(task, index)
        if msg != "" </span><span class="cov0" title="0">{
                return msg
        }</span>

        <span class="cov8" title="1">return ""</span>
}

func validateK8sPodNameLength(podName string) string <span class="cov8" title="1">{
        if errMsgs := validation.IsQualifiedName(podName); len(errMsgs) &gt; 0 </span><span class="cov0" title="0">{
                return fmt.Sprintf("create pod with name %s validate failed %v;", podName, errMsgs)
        }</span>
        <span class="cov8" title="1">return ""</span>
}

func validateJobName(job *v1alpha1.Job) string <span class="cov8" title="1">{
        if errMsgs := validation.IsQualifiedName(job.Name); len(errMsgs) &gt; 0 </span><span class="cov0" title="0">{
                return fmt.Sprintf("create job with name %s validate failed %v", job.Name, errMsgs)
        }</span>
        <span class="cov8" title="1">return ""</span>
}

func validateTaskTopoPolicy(task v1alpha1.TaskSpec, index int) string <span class="cov8" title="1">{
        if task.TopologyPolicy == "" || task.TopologyPolicy == v1alpha1.None </span><span class="cov8" title="1">{
                return ""
        }</span>

        <span class="cov8" title="1">template := task.Template.DeepCopy()

        for id, container := range template.Spec.Containers </span><span class="cov8" title="1">{
                if len(container.Resources.Requests) == 0 </span><span class="cov8" title="1">{
                        template.Spec.Containers[id].Resources.Requests = container.Resources.Limits.DeepCopy()
                }</span>
        }

        <span class="cov8" title="1">for id, container := range template.Spec.InitContainers </span><span class="cov0" title="0">{
                if len(container.Resources.Requests) == 0 </span><span class="cov0" title="0">{
                        template.Spec.InitContainers[id].Resources.Requests = container.Resources.Limits.DeepCopy()
                }</span>
        }

        <span class="cov8" title="1">pod := &amp;v1.Pod{
                Spec: template.Spec,
        }

        if v1qos.GetPodQOS(pod) != v1.PodQOSGuaranteed </span><span class="cov8" title="1">{
                return fmt.Sprintf("spec.task[%d] isn't Guaranteed pod, kind=%v", index, v1qos.GetPodQOS(pod))
        }</span>

        <span class="cov8" title="1">for id, container := range append(template.Spec.Containers, template.Spec.InitContainers...) </span><span class="cov8" title="1">{
                requestNum := guaranteedCPUs(container)
                if requestNum == 0 </span><span class="cov8" title="1">{
                        return fmt.Sprintf("the cpu request isn't  an integer in spec.task[%d] container[%d].",
                                index, id)
                }</span>
        }

        <span class="cov8" title="1">return ""</span>
}

func guaranteedCPUs(container v1.Container) int <span class="cov8" title="1">{
        cpuQuantity := container.Resources.Requests[v1.ResourceCPU]
        if cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() </span><span class="cov8" title="1">{
                return 0
        }</span>

        <span class="cov8" title="1">return int(cpuQuantity.Value())</span>
}
</pre>
		
		<pre class="file" id="file89" style="display: none">/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validate

import (
        "fmt"

        "github.com/hashicorp/go-multierror"

        "k8s.io/apimachinery/pkg/util/validation/field"
        "k8s.io/kubernetes/pkg/apis/core/validation"

        batchv1alpha1 "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1"
)

// policyEventMap defines all policy events and whether to allow external use.
var policyEventMap = map[busv1alpha1.Event]bool{
        busv1alpha1.AnyEvent:           true,
        busv1alpha1.PodFailedEvent:     true,
        busv1alpha1.PodEvictedEvent:    true,
        busv1alpha1.JobUnknownEvent:    true,
        busv1alpha1.TaskCompletedEvent: true,
        busv1alpha1.TaskFailedEvent:    true,
        busv1alpha1.OutOfSyncEvent:     false,
        busv1alpha1.CommandIssuedEvent: false,
        busv1alpha1.JobUpdatedEvent:    true,
}

// policyActionMap defines all policy actions and whether to allow external use.
var policyActionMap = map[busv1alpha1.Action]bool{
        busv1alpha1.AbortJobAction:     true,
        busv1alpha1.RestartJobAction:   true,
        busv1alpha1.RestartTaskAction:  true,
        busv1alpha1.TerminateJobAction: true,
        busv1alpha1.CompleteJobAction:  true,
        busv1alpha1.ResumeJobAction:    true,
        busv1alpha1.SyncJobAction:      false,
        busv1alpha1.EnqueueAction:      false,
        busv1alpha1.SyncQueueAction:    false,
        busv1alpha1.OpenQueueAction:    false,
        busv1alpha1.CloseQueueAction:   false,
}

func validatePolicies(policies []batchv1alpha1.LifecyclePolicy, fldPath *field.Path) error <span class="cov8" title="1">{
        var err error
        policyEvents := map[busv1alpha1.Event]struct{}{}
        exitCodes := map[int32]struct{}{}

        for _, policy := range policies </span><span class="cov8" title="1">{
                if (policy.Event != "" || len(policy.Events) != 0) &amp;&amp; policy.ExitCode != nil </span><span class="cov8" title="1">{
                        err = multierror.Append(err, fmt.Errorf("must not specify event and exitCode simultaneously"))
                        break</span>
                }

                <span class="cov8" title="1">if policy.Event == "" &amp;&amp; len(policy.Events) == 0 &amp;&amp; policy.ExitCode == nil </span><span class="cov8" title="1">{
                        err = multierror.Append(err, fmt.Errorf("either event and exitCode should be specified"))
                        break</span>
                }

                <span class="cov8" title="1">if len(policy.Event) != 0 || len(policy.Events) != 0 </span><span class="cov8" title="1">{
                        bFlag := false
                        policyEventsList := getEventList(policy)
                        for _, event := range policyEventsList </span><span class="cov8" title="1">{
                                if allow, ok := policyEventMap[event]; !ok || !allow </span><span class="cov8" title="1">{
                                        err = multierror.Append(err, field.Invalid(fldPath, event, "invalid policy event"))
                                        bFlag = true
                                        break</span>
                                }

                                <span class="cov8" title="1">if allow, ok := policyActionMap[policy.Action]; !ok || !allow </span><span class="cov8" title="1">{
                                        err = multierror.Append(err, field.Invalid(fldPath, policy.Action, "invalid policy action"))
                                        bFlag = true
                                        break</span>
                                }
                                <span class="cov8" title="1">if _, found := policyEvents[event]; found </span><span class="cov8" title="1">{
                                        err = multierror.Append(err, fmt.Errorf("duplicate event %v  across different policy", event))
                                        bFlag = true
                                        break</span>
                                } else<span class="cov8" title="1"> {
                                        policyEvents[event] = struct{}{}
                                }</span>
                        }
                        <span class="cov8" title="1">if bFlag </span><span class="cov8" title="1">{
                                break</span>
                        }
                } else<span class="cov8" title="1"> {
                        if *policy.ExitCode == 0 </span><span class="cov8" title="1">{
                                err = multierror.Append(err, fmt.Errorf("0 is not a valid error code"))
                                break</span>
                        }
                        <span class="cov8" title="1">if _, found := exitCodes[*policy.ExitCode]; found </span><span class="cov8" title="1">{
                                err = multierror.Append(err, fmt.Errorf("duplicate exitCode %v", *policy.ExitCode))
                                break</span>
                        } else<span class="cov8" title="1"> {
                                exitCodes[*policy.ExitCode] = struct{}{}
                        }</span>
                }
        }

        <span class="cov8" title="1">if _, found := policyEvents[busv1alpha1.AnyEvent]; found &amp;&amp; len(policyEvents) &gt; 1 </span><span class="cov8" title="1">{
                err = multierror.Append(err, fmt.Errorf("if there's * here, no other policy should be here"))
        }</span>

        <span class="cov8" title="1">return err</span>
}

func getEventList(policy batchv1alpha1.LifecyclePolicy) []busv1alpha1.Event <span class="cov8" title="1">{
        policyEventsList := policy.Events
        if len(policy.Event) &gt; 0 </span><span class="cov8" title="1">{
                policyEventsList = append(policyEventsList, policy.Event)
        }</span>
        <span class="cov8" title="1">uniquePolicyEventlist := removeDuplicates(policyEventsList)
        return uniquePolicyEventlist</span>
}

func removeDuplicates(eventList []busv1alpha1.Event) []busv1alpha1.Event <span class="cov8" title="1">{
        keys := make(map[busv1alpha1.Event]bool)
        list := []busv1alpha1.Event{}
        for _, val := range eventList </span><span class="cov8" title="1">{
                if _, value := keys[val]; !value </span><span class="cov8" title="1">{
                        keys[val] = true
                        list = append(list, val)
                }</span>
        }
        <span class="cov8" title="1">return list</span>
}

func getValidEvents() []busv1alpha1.Event <span class="cov8" title="1">{
        var events []busv1alpha1.Event
        for e, allow := range policyEventMap </span><span class="cov8" title="1">{
                if allow </span><span class="cov8" title="1">{
                        events = append(events, e)
                }</span>
        }

        <span class="cov8" title="1">return events</span>
}

func getValidActions() []busv1alpha1.Action <span class="cov8" title="1">{
        var actions []busv1alpha1.Action
        for a, allow := range policyActionMap </span><span class="cov8" title="1">{
                if allow </span><span class="cov8" title="1">{
                        actions = append(actions, a)
                }</span>
        }

        <span class="cov8" title="1">return actions</span>
}

// validateIO validates IO configuration.
func validateIO(volumes []batchv1alpha1.VolumeSpec) error <span class="cov8" title="1">{
        volumeMap := map[string]bool{}
        for _, volume := range volumes </span><span class="cov8" title="1">{
                if len(volume.MountPath) == 0 </span><span class="cov8" title="1">{
                        return fmt.Errorf(" mountPath is required;")
                }</span>
                <span class="cov8" title="1">if _, found := volumeMap[volume.MountPath]; found </span><span class="cov8" title="1">{
                        return fmt.Errorf(" duplicated mountPath: %s;", volume.MountPath)
                }</span>
                <span class="cov8" title="1">if volume.VolumeClaim == nil &amp;&amp; volume.VolumeClaimName == "" </span><span class="cov8" title="1">{
                        return fmt.Errorf(" either VolumeClaim or VolumeClaimName must be specified;")
                }</span>
                <span class="cov8" title="1">if len(volume.VolumeClaimName) != 0 </span><span class="cov8" title="1">{
                        if volume.VolumeClaim != nil </span><span class="cov0" title="0">{
                                return fmt.Errorf("conflict: If you want to use an existing PVC, just specify VolumeClaimName." +
                                        "If you want to create a new PVC, you do not need to specify VolumeClaimName")
                        }</span>
                        <span class="cov8" title="1">if errMsgs := validation.ValidatePersistentVolumeName(volume.VolumeClaimName, false); len(errMsgs) &gt; 0 </span><span class="cov0" title="0">{
                                return fmt.Errorf("invalid VolumeClaimName %s : %v", volume.VolumeClaimName, errMsgs)
                        }</span>
                }

                <span class="cov8" title="1">volumeMap[volume.MountPath] = true</span>
        }
        <span class="cov8" title="1">return nil</span>
}
</pre>
		
		<pre class="file" id="file90" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mutate

import (
        "github.com/imdario/mergo"
        "gopkg.in/yaml.v2"
        "k8s.io/klog"

        v1 "k8s.io/api/core/v1"

        wkconfig "volcano.sh/volcano/pkg/webhooks/config"
)

type annotationResGroup struct{}

const (
        // defaultAnnotationKey: default annotation key
        defaultAnnotationKey = "volcano.sh/resource-group"
)

// NewAnnotationResGroup create a new structure
func NewAnnotationResGroup() ResGroup <span class="cov8" title="1">{
        return &amp;annotationResGroup{}
}</span>

// getAnnotation get annotations from the resource group
func getAnnotation(resGroupConfig wkconfig.ResGroupConfig) map[string]string <span class="cov8" title="1">{
        annotations := make(map[string]string)
        for _, val := range resGroupConfig.Object.Value </span><span class="cov8" title="1">{
                tmp := make(map[string]string)
                err := yaml.Unmarshal([]byte(val), &amp;tmp)
                if err != nil </span><span class="cov0" title="0">{
                        continue</span>
                }

                <span class="cov8" title="1">if err := mergo.Merge(&amp;annotations, &amp;tmp); err != nil </span><span class="cov0" title="0">{
                        klog.Errorf("annotations merge failed, err=%v", err)
                        continue</span>
                }
        }

        <span class="cov8" title="1">return annotations</span>
}

// IsBelongResGroup adjust whether pod is belong to the resource group
func (resGroup *annotationResGroup) IsBelongResGroup(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) bool <span class="cov8" title="1">{
        if resGroupConfig.Object.Key != "" &amp;&amp; resGroupConfig.Object.Key != "annotation" </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov8" title="1">annotations := getAnnotation(resGroupConfig)
        klog.V(3).Infof("annotations : %v", annotations)
        for key, annotation := range annotations </span><span class="cov8" title="1">{
                if pod.Annotations[key] == annotation </span><span class="cov8" title="1">{
                        return true
                }</span>
        }

        <span class="cov8" title="1">if resGroupConfig.Object.Key == "" &amp;&amp; pod.Annotations[defaultAnnotationKey] == resGroupConfig.ResourceGroup </span><span class="cov8" title="1">{
                return true
        }</span>

        <span class="cov8" title="1">return false</span>
}
</pre>
		
		<pre class="file" id="file91" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mutate

import (
        v1 "k8s.io/api/core/v1"

        wkconfig "volcano.sh/volcano/pkg/webhooks/config"
)

// ResGroup interface for resource group
type ResGroup interface {
        IsBelongResGroup(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) bool
}

// GetResGroup return the interface besed on resourceGroup.Object.Key
func GetResGroup(resourceGroup wkconfig.ResGroupConfig) ResGroup <span class="cov8" title="1">{
        switch resourceGroup.Object.Key </span>{
        case "namespace":<span class="cov8" title="1">
                return NewNamespaceResGroup()</span>
        case "annotation":<span class="cov8" title="1">
                return NewAnnotationResGroup()</span>
        }
        <span class="cov8" title="1">return NewAnnotationResGroup()</span>
}
</pre>
		
		<pre class="file" id="file92" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mutate

import (
        "encoding/json"
        "fmt"

        "k8s.io/api/admission/v1beta1"
        whv1beta1 "k8s.io/api/admissionregistration/v1beta1"
        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        wkconfig "volcano.sh/volcano/pkg/webhooks/config"
        "volcano.sh/volcano/pkg/webhooks/router"
        "volcano.sh/volcano/pkg/webhooks/schema"
        "volcano.sh/volcano/pkg/webhooks/util"
)

// patchOperation define the patch operation structure
type patchOperation struct {
        Op    string      `json:"op"`
        Path  string      `json:"path"`
        Value interface{} `json:"value,omitempty"`
}

// init register mutate pod
func init() <span class="cov8" title="1">{
        router.RegisterAdmission(service)
}</span>

var service = &amp;router.AdmissionService{
        Path:   "/pods/mutate",
        Func:   Pods,
        Config: config,
        MutatingConfig: &amp;whv1beta1.MutatingWebhookConfiguration{
                Webhooks: []whv1beta1.MutatingWebhook{{
                        Name: "mutatepod.volcano.sh",
                        Rules: []whv1beta1.RuleWithOperations{
                                {
                                        Operations: []whv1beta1.OperationType{whv1beta1.Create},
                                        Rule: whv1beta1.Rule{
                                                APIGroups:   []string{""},
                                                APIVersions: []string{"v1"},
                                                Resources:   []string{"pods"},
                                        },
                                },
                        },
                }},
        },
}

var config = &amp;router.AdmissionServiceConfig{}

// Pods mutate pods.
func Pods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse <span class="cov0" title="0">{
        klog.V(3).Infof("mutating pods -- %s", ar.Request.Operation)
        pod, err := schema.DecodePod(ar.Request.Object, ar.Request.Resource)
        if err != nil </span><span class="cov0" title="0">{
                return util.ToAdmissionResponse(err)
        }</span>

        <span class="cov0" title="0">if pod.Namespace == "" </span><span class="cov0" title="0">{
                pod.Namespace = ar.Request.Namespace
        }</span>

        <span class="cov0" title="0">var patchBytes []byte
        switch ar.Request.Operation </span>{
        case v1beta1.Create:<span class="cov0" title="0">
                patchBytes, _ = createPatch(pod)</span>
        default:<span class="cov0" title="0">
                err = fmt.Errorf("expect operation to be 'CREATE' ")
                return util.ToAdmissionResponse(err)</span>
        }

        <span class="cov0" title="0">reviewResponse := v1beta1.AdmissionResponse{
                Allowed: true,
                Patch:   patchBytes,
        }
        pt := v1beta1.PatchTypeJSONPatch
        reviewResponse.PatchType = &amp;pt

        return &amp;reviewResponse</span>
}

// createPatch patch pod
func createPatch(pod *v1.Pod) ([]byte, error) <span class="cov8" title="1">{
        if config.ConfigData == nil </span><span class="cov0" title="0">{
                klog.V(5).Infof("admission configuration is empty.")
                return nil, nil
        }</span>

        <span class="cov8" title="1">var patch []patchOperation
        config.ConfigData.Lock()
        defer config.ConfigData.Unlock()

        for _, resourceGroup := range config.ConfigData.ResGroupsConfig </span><span class="cov8" title="1">{
                klog.V(3).Infof("resourceGroup %s", resourceGroup.ResourceGroup)
                group := GetResGroup(resourceGroup)
                if !group.IsBelongResGroup(pod, resourceGroup) </span><span class="cov8" title="1">{
                        continue</span>
                }

                <span class="cov8" title="1">patchLabel := patchLabels(pod, resourceGroup)
                if patchLabel != nil </span><span class="cov8" title="1">{
                        patch = append(patch, *patchLabel)
                }</span>

                <span class="cov8" title="1">patchToleration := patchTaintToleration(pod, resourceGroup)
                if patchToleration != nil </span><span class="cov8" title="1">{
                        patch = append(patch, *patchToleration)
                }</span>
                <span class="cov8" title="1">patchScheduler := patchSchedulerName(resourceGroup)
                if patchScheduler != nil </span><span class="cov8" title="1">{
                        patch = append(patch, *patchScheduler)
                }</span>

                <span class="cov8" title="1">klog.V(5).Infof("pod patch %v", patch)
                return json.Marshal(patch)</span>
        }

        <span class="cov8" title="1">return json.Marshal(patch)</span>
}

// patchLabels patch label
func patchLabels(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) *patchOperation <span class="cov8" title="1">{
        if len(resGroupConfig.Labels) == 0 </span><span class="cov0" title="0">{
                return nil
        }</span>

        <span class="cov8" title="1">nodeSelector := make(map[string]string)
        for key, label := range pod.Spec.NodeSelector </span><span class="cov0" title="0">{
                nodeSelector[key] = label
        }</span>

        <span class="cov8" title="1">for key, label := range resGroupConfig.Labels </span><span class="cov8" title="1">{
                nodeSelector[key] = label
        }</span>

        <span class="cov8" title="1">return &amp;patchOperation{Op: "add", Path: "/spec/nodeSelector", Value: nodeSelector}</span>
}

// patchTaintToleration patch taint toleration
func patchTaintToleration(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) *patchOperation <span class="cov8" title="1">{
        if len(resGroupConfig.Tolerations) == 0 </span><span class="cov8" title="1">{
                return nil
        }</span>

        <span class="cov8" title="1">var dst []v1.Toleration
        dst = append(dst, pod.Spec.Tolerations...)
        dst = append(dst, resGroupConfig.Tolerations...)

        return &amp;patchOperation{Op: "add", Path: "/spec/tolerations", Value: dst}</span>
}

// patchSchedulerName patch scheduler
func patchSchedulerName(resGroupConfig wkconfig.ResGroupConfig) *patchOperation <span class="cov8" title="1">{
        if resGroupConfig.SchedulerName == "" </span><span class="cov0" title="0">{
                return nil
        }</span>

        <span class="cov8" title="1">return &amp;patchOperation{Op: "add", Path: "/spec/schedulerName", Value: resGroupConfig.SchedulerName}</span>
}
</pre>
		
		<pre class="file" id="file93" style="display: none">/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mutate

import (
        v1 "k8s.io/api/core/v1"

        wkconfig "volcano.sh/volcano/pkg/webhooks/config"
)

type namespaceResGroup struct{}

// NewNamespaceResGroup create a new structure
func NewNamespaceResGroup() ResGroup <span class="cov8" title="1">{
        return &amp;namespaceResGroup{}
}</span>

// IsBelongResGroup adjust whether pod is belong to the resource group
func (resGroup *namespaceResGroup) IsBelongResGroup(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) bool <span class="cov8" title="1">{
        if resGroupConfig.Object.Key != "namespace" </span><span class="cov0" title="0">{
                return false
        }</span>

        <span class="cov8" title="1">for _, val := range resGroupConfig.Object.Value </span><span class="cov8" title="1">{
                if pod.Namespace == val </span><span class="cov8" title="1">{
                        return true
                }</span>
        }

        <span class="cov8" title="1">return false</span>
}
</pre>
		
		<pre class="file" id="file94" style="display: none">/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validate

import (
        "context"
        "fmt"
        "strconv"

        "strings"

        "k8s.io/api/admission/v1beta1"
        whv1beta1 "k8s.io/api/admissionregistration/v1beta1"
        v1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/util/intstr"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/helpers"
        vcv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/webhooks/router"
        "volcano.sh/volcano/pkg/webhooks/schema"
        "volcano.sh/volcano/pkg/webhooks/util"
)

func init() <span class="cov8" title="1">{
        router.RegisterAdmission(service)
}</span>

var service = &amp;router.AdmissionService{
        Path: "/pods/validate",
        Func: AdmitPods,

        Config: config,

        ValidatingConfig: &amp;whv1beta1.ValidatingWebhookConfiguration{
                Webhooks: []whv1beta1.ValidatingWebhook{{
                        Name: "validatepod.volcano.sh",
                        Rules: []whv1beta1.RuleWithOperations{
                                {
                                        Operations: []whv1beta1.OperationType{whv1beta1.Create},
                                        Rule: whv1beta1.Rule{
                                                APIGroups:   []string{""},
                                                APIVersions: []string{"v1"},
                                                Resources:   []string{"pods"},
                                        },
                                },
                        },
                }},
        },
}

var config = &amp;router.AdmissionServiceConfig{}

// AdmitPods is to admit pods and return response.
func AdmitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse <span class="cov0" title="0">{
        klog.V(3).Infof("admitting pods -- %s", ar.Request.Operation)

        pod, err := schema.DecodePod(ar.Request.Object, ar.Request.Resource)
        if err != nil </span><span class="cov0" title="0">{
                return util.ToAdmissionResponse(err)
        }</span>

        <span class="cov0" title="0">var msg string
        reviewResponse := v1beta1.AdmissionResponse{}
        reviewResponse.Allowed = true

        switch ar.Request.Operation </span>{
        case v1beta1.Create:<span class="cov0" title="0">
                msg = validatePod(pod, &amp;reviewResponse)</span>
        default:<span class="cov0" title="0">
                err := fmt.Errorf("expect operation to be 'CREATE'")
                return util.ToAdmissionResponse(err)</span>
        }

        <span class="cov0" title="0">if !reviewResponse.Allowed </span><span class="cov0" title="0">{
                reviewResponse.Result = &amp;metav1.Status{Message: strings.TrimSpace(msg)}
        }</span>
        <span class="cov0" title="0">return &amp;reviewResponse</span>
}

/*
allow pods to create when
1. schedulerName of pod isn't volcano
2. pod has Podgroup whose phase isn't Pending
3. normal pods whose schedulerName is volcano don't have podgroup.
4. check pod budget annotations configure
*/
func validatePod(pod *v1.Pod, reviewResponse *v1beta1.AdmissionResponse) string <span class="cov8" title="1">{
        if pod.Spec.SchedulerName != config.SchedulerName </span><span class="cov8" title="1">{
                return ""
        }</span>

        <span class="cov8" title="1">pgName := ""
        msg := ""

        // vc-job, SN == volcano
        if pod.Annotations != nil </span><span class="cov8" title="1">{
                pgName = pod.Annotations[vcv1beta1.KubeGroupNameAnnotationKey]
        }</span>
        <span class="cov8" title="1">if pgName != "" </span><span class="cov8" title="1">{
                if err := checkPGPhase(pod, pgName, true); err != nil </span><span class="cov8" title="1">{
                        msg = err.Error()
                        reviewResponse.Allowed = false
                }</span>
                <span class="cov8" title="1">return msg</span>
        }

        // normal pod, SN == volcano
        <span class="cov8" title="1">pgName = helpers.GeneratePodgroupName(pod)
        if err := checkPGPhase(pod, pgName, false); err != nil </span><span class="cov8" title="1">{
                msg = err.Error()
                reviewResponse.Allowed = false
        }</span>

        // check pod annotatations
        <span class="cov8" title="1">if err := validateAnnotation(pod); err != nil </span><span class="cov0" title="0">{
                msg = err.Error()
                reviewResponse.Allowed = false
        }</span>

        <span class="cov8" title="1">return msg</span>
}

func checkPGPhase(pod *v1.Pod, pgName string, isVCJob bool) error <span class="cov8" title="1">{
        pg, err := config.VolcanoClient.SchedulingV1beta1().PodGroups(pod.Namespace).Get(context.TODO(), pgName, metav1.GetOptions{})
        if err != nil </span><span class="cov8" title="1">{
                if isVCJob || (!isVCJob &amp;&amp; !apierrors.IsNotFound(err)) </span><span class="cov8" title="1">{
                        return fmt.Errorf("failed to get PodGroup for pod &lt;%s/%s&gt;: %v", pod.Namespace, pod.Name, err)
                }</span>
                <span class="cov0" title="0">return nil</span>
        }
        <span class="cov8" title="1">if pg.Status.Phase != vcv1beta1.PodGroupPending </span><span class="cov0" title="0">{
                return nil
        }</span>
        <span class="cov8" title="1">return fmt.Errorf("failed to create pod &lt;%s/%s&gt; as the podgroup phase is Pending",
                pod.Namespace, pod.Name)</span>
}

func validateAnnotation(pod *v1.Pod) error <span class="cov8" title="1">{
        num := 0
        if len(pod.Annotations) &gt; 0 </span><span class="cov0" title="0">{
                keys := []string{
                        vcv1beta1.JDBMinAvailable,
                        vcv1beta1.JDBMaxUnavailable,
                }
                for _, key := range keys </span><span class="cov0" title="0">{
                        if value, found := pod.Annotations[key]; found </span><span class="cov0" title="0">{
                                num++
                                if err := validateIntPercentageStr(key, value); err != nil </span><span class="cov0" title="0">{
                                        recordEvent(err)
                                        return err
                                }</span>
                        }
                }
                <span class="cov0" title="0">if num &gt; 1 </span><span class="cov0" title="0">{
                        return fmt.Errorf("not allow configure multiple annotations &lt;%v&gt; at same time", keys)
                }</span>
        }
        <span class="cov8" title="1">return nil</span>
}

func recordEvent(err error) <span class="cov0" title="0">{
        config.Recorder.Eventf(nil, v1.EventTypeWarning, "Admit", "Create pod failed due to %v", err)
}</span>

func validateIntPercentageStr(key, value string) error <span class="cov0" title="0">{
        tmp := intstr.Parse(value)
        switch tmp.Type </span>{
        case intstr.Int:<span class="cov0" title="0">
                if tmp.IntValue() &lt;= 0 </span><span class="cov0" title="0">{
                        return fmt.Errorf("invalid value &lt;%q&gt; for %v, it must be a positive integer", value, key)
                }</span>
                <span class="cov0" title="0">return nil</span>
        case intstr.String:<span class="cov0" title="0">
                s := strings.Replace(tmp.StrVal, "%", "", -1)
                v, err := strconv.Atoi(s)
                if err != nil </span><span class="cov0" title="0">{
                        return fmt.Errorf("invalid value %v for %v", err, key)
                }</span>
                <span class="cov0" title="0">if v &lt;= 0 || v &gt;= 100 </span><span class="cov0" title="0">{
                        return fmt.Errorf("invalid value &lt;%q&gt; for %v, it must be a valid percentage which between 1%% ~ 99%%", tmp.StrVal, key)
                }</span>
                <span class="cov0" title="0">return nil</span>
        }
        <span class="cov0" title="0">return fmt.Errorf("invalid type: neither int nor percentage for %v", key)</span>
}
</pre>
		
		<pre class="file" id="file95" style="display: none">/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mutate

import (
        "encoding/json"
        "fmt"
        "strings"

        "k8s.io/api/admission/v1beta1"
        whv1beta1 "k8s.io/api/admissionregistration/v1beta1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/klog"

        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/webhooks/router"
        "volcano.sh/volcano/pkg/webhooks/schema"
        "volcano.sh/volcano/pkg/webhooks/util"
)

func init() <span class="cov8" title="1">{
        router.RegisterAdmission(service)
}</span>

var service = &amp;router.AdmissionService{
        Path: "/queues/mutate",
        Func: Queues,

        MutatingConfig: &amp;whv1beta1.MutatingWebhookConfiguration{
                Webhooks: []whv1beta1.MutatingWebhook{{
                        Name: "mutatequeue.volcano.sh",
                        Rules: []whv1beta1.RuleWithOperations{
                                {
                                        Operations: []whv1beta1.OperationType{whv1beta1.Create},
                                        Rule: whv1beta1.Rule{
                                                APIGroups:   []string{schedulingv1beta1.SchemeGroupVersion.Group},
                                                APIVersions: []string{schedulingv1beta1.SchemeGroupVersion.Version},
                                                Resources:   []string{"queues"},
                                        },
                                },
                        },
                }},
        },
}

type patchOperation struct {
        Op    string      `json:"op"`
        Path  string      `json:"path"`
        Value interface{} `json:"value,omitempty"`
}

// Queues mutate queues.
func Queues(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse <span class="cov8" title="1">{
        klog.V(3).Infof("Mutating %s queue %s.", ar.Request.Operation, ar.Request.Name)

        queue, err := schema.DecodeQueue(ar.Request.Object, ar.Request.Resource)
        if err != nil </span><span class="cov0" title="0">{
                return util.ToAdmissionResponse(err)
        }</span>

        <span class="cov8" title="1">var patchBytes []byte
        switch ar.Request.Operation </span>{
        case v1beta1.Create:<span class="cov8" title="1">
                patchBytes, err = createQueuePatch(queue)</span>
        default:<span class="cov8" title="1">
                return util.ToAdmissionResponse(fmt.Errorf("invalid operation `%s`, "+
                        "expect operation to be `CREATE`", ar.Request.Operation))</span>
        }

        <span class="cov8" title="1">if err != nil </span><span class="cov0" title="0">{
                return &amp;v1beta1.AdmissionResponse{
                        Allowed: false,
                        Result:  &amp;metav1.Status{Message: err.Error()},
                }
        }</span>

        <span class="cov8" title="1">pt := v1beta1.PatchTypeJSONPatch
        return &amp;v1beta1.AdmissionResponse{
                Allowed:   true,
                Patch:     patchBytes,
                PatchType: &amp;pt,
        }</span>
}

func createQueuePatch(queue *schedulingv1beta1.Queue) ([]byte, error) <span class="cov8" title="1">{
        var patch []patchOperation

        // add root node if the root node not specified
        hierarchy := queue.Annotations[schedulingv1beta1.KubeHierarchyAnnotationKey]
        hierarchicalWeights := queue.Annotations[schedulingv1beta1.KubeHierarchyWeightAnnotationKey]

        if hierarchy != "" &amp;&amp; hierarchicalWeights != "" &amp;&amp; !strings.HasPrefix(hierarchy, "root") </span><span class="cov8" title="1">{
                // based on https://tools.ietf.org/html/rfc6901#section-3
                // escape "/" with "~1"
                patch = append(patch, patchOperation{
                        Op:    "add",
                        Path:  fmt.Sprintf("/metadata/annotations/%s", strings.ReplaceAll(schedulingv1beta1.KubeHierarchyAnnotationKey, "/", "~1")),
                        Value: fmt.Sprintf("root/%s", hierarchy),
                })
                patch = append(patch, patchOperation{
                        Op:    "add",
                        Path:  fmt.Sprintf("/metadata/annotations/%s", strings.ReplaceAll(schedulingv1beta1.KubeHierarchyWeightAnnotationKey, "/", "~1")),
                        Value: fmt.Sprintf("1/%s", hierarchicalWeights),
                })
        }</span>

        <span class="cov8" title="1">trueValue := true
        if queue.Spec.Reclaimable == nil </span><span class="cov8" title="1">{
                patch = append(patch, patchOperation{
                        Op:    "add",
                        Path:  "/spec/reclaimable",
                        Value: &amp;trueValue,
                })
        }</span>

        <span class="cov8" title="1">defaultWeight := 1
        if queue.Spec.Weight == 0 </span><span class="cov0" title="0">{
                patch = append(patch, patchOperation{
                        Op:    "add",
                        Path:  "/spec/weight",
                        Value: &amp;defaultWeight,
                })
        }</span>

        <span class="cov8" title="1">return json.Marshal(patch)</span>
}
</pre>
		
		<pre class="file" id="file96" style="display: none">/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validate

import (
        "context"
        "fmt"
        "strconv"
        "strings"

        "k8s.io/api/admission/v1beta1"
        whv1beta1 "k8s.io/api/admissionregistration/v1beta1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/util/validation/field"
        "k8s.io/klog"

        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/webhooks/router"
        "volcano.sh/volcano/pkg/webhooks/schema"
        "volcano.sh/volcano/pkg/webhooks/util"
)

func init() <span class="cov8" title="1">{
        router.RegisterAdmission(service)
}</span>

var service = &amp;router.AdmissionService{
        Path: "/queues/validate",
        Func: AdmitQueues,

        Config: config,

        ValidatingConfig: &amp;whv1beta1.ValidatingWebhookConfiguration{
                Webhooks: []whv1beta1.ValidatingWebhook{{
                        Name: "validatequeue.volcano.sh",
                        Rules: []whv1beta1.RuleWithOperations{
                                {
                                        Operations: []whv1beta1.OperationType{whv1beta1.Create, whv1beta1.Update, whv1beta1.Delete},
                                        Rule: whv1beta1.Rule{
                                                APIGroups:   []string{schedulingv1beta1.SchemeGroupVersion.Group},
                                                APIVersions: []string{schedulingv1beta1.SchemeGroupVersion.Version},
                                                Resources:   []string{"queues"},
                                        },
                                },
                        },
                }},
        },
}

var config = &amp;router.AdmissionServiceConfig{}

// AdmitQueues is to admit queues and return response.
func AdmitQueues(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse <span class="cov8" title="1">{
        klog.V(3).Infof("Admitting %s queue %s.", ar.Request.Operation, ar.Request.Name)

        queue, err := schema.DecodeQueue(ar.Request.Object, ar.Request.Resource)
        if err != nil </span><span class="cov0" title="0">{
                return util.ToAdmissionResponse(err)
        }</span>

        <span class="cov8" title="1">switch ar.Request.Operation </span>{
        case v1beta1.Create, v1beta1.Update:<span class="cov8" title="1">
                err = validateQueue(queue)</span>
        case v1beta1.Delete:<span class="cov8" title="1">
                err = validateQueueDeleting(ar.Request.Name)</span>
        default:<span class="cov8" title="1">
                return util.ToAdmissionResponse(fmt.Errorf("invalid operation `%s`, "+
                        "expect operation to be `CREATE`, `UPDATE` or `DELETE`", ar.Request.Operation))</span>
        }

        <span class="cov8" title="1">if err != nil </span><span class="cov8" title="1">{
                return &amp;v1beta1.AdmissionResponse{
                        Allowed: false,
                        Result:  &amp;metav1.Status{Message: err.Error()},
                }
        }</span>

        <span class="cov8" title="1">return &amp;v1beta1.AdmissionResponse{
                Allowed: true,
        }</span>
}

func validateQueue(queue *schedulingv1beta1.Queue) error <span class="cov8" title="1">{
        errs := field.ErrorList{}
        resourcePath := field.NewPath("requestBody")

        errs = append(errs, validateStateOfQueue(queue.Status.State, resourcePath.Child("spec").Child("state"))...)
        errs = append(errs, validateWeightOfQueue(queue.Spec.Weight, resourcePath.Child("spec").Child("weight"))...)
        errs = append(errs, validateHierarchicalAttributes(queue, resourcePath.Child("metadata").Child("annotations"))...)

        if len(errs) &gt; 0 </span><span class="cov8" title="1">{
                return errs.ToAggregate()
        }</span>

        <span class="cov8" title="1">return nil</span>
}
func validateHierarchicalAttributes(queue *schedulingv1beta1.Queue, fldPath *field.Path) field.ErrorList <span class="cov8" title="1">{
        errs := field.ErrorList{}
        hierarchy := queue.Annotations[schedulingv1beta1.KubeHierarchyAnnotationKey]
        hierarchicalWeights := queue.Annotations[schedulingv1beta1.KubeHierarchyWeightAnnotationKey]
        if hierarchy != "" || hierarchicalWeights != "" </span><span class="cov8" title="1">{
                paths := strings.Split(hierarchy, "/")
                weights := strings.Split(hierarchicalWeights, "/")
                // path length must be the same with weights length
                if len(paths) != len(weights) </span><span class="cov8" title="1">{
                        return append(errs, field.Invalid(fldPath, hierarchy,
                                fmt.Sprintf("%s must have the same length with %s",
                                        schedulingv1beta1.KubeHierarchyAnnotationKey,
                                        schedulingv1beta1.KubeHierarchyWeightAnnotationKey,
                                )))
                }</span>

                // check weights format
                <span class="cov8" title="1">for _, weight := range weights </span><span class="cov8" title="1">{
                        weightFloat, err := strconv.ParseFloat(weight, 64)
                        if err != nil </span><span class="cov8" title="1">{
                                return append(errs, field.Invalid(fldPath, hierarchicalWeights,
                                        fmt.Sprintf("%s in the %s is invalid number: %v",
                                                weight, hierarchicalWeights, err,
                                        )))
                        }</span>
                        <span class="cov8" title="1">if weightFloat &lt;= 0 </span><span class="cov8" title="1">{
                                return append(errs, field.Invalid(fldPath, hierarchicalWeights,
                                        fmt.Sprintf("%s in the %s must be larger than 0",
                                                weight, hierarchicalWeights,
                                        )))
                        }</span>
                }

                // The node is not allowed to be in the sub path of a node.
                // For example, a queue with "root/sci" conflicts with a queue with "root/sci/dev"
                <span class="cov8" title="1">queueList, err := config.VolcanoClient.SchedulingV1beta1().Queues().List(context.TODO(), metav1.ListOptions{})
                if err != nil </span><span class="cov0" title="0">{
                        return append(errs, field.Invalid(fldPath, hierarchy,
                                fmt.Sprintf("checking %s, list queues failed: %v",
                                        schedulingv1beta1.KubeHierarchyAnnotationKey,
                                        err,
                                )))
                }</span>
                <span class="cov8" title="1">for _, queueInTree := range queueList.Items </span><span class="cov8" title="1">{
                        hierarchyInTree := queueInTree.Annotations[schedulingv1beta1.KubeHierarchyAnnotationKey]
                        if hierarchyInTree != "" &amp;&amp; queue.Name != queueInTree.Name &amp;&amp;
                                strings.HasPrefix(hierarchyInTree, hierarchy) </span><span class="cov8" title="1">{
                                return append(errs, field.Invalid(fldPath, hierarchy,
                                        fmt.Sprintf("%s is not allowed to be in the sub path of %s of queue %s",
                                                hierarchy, hierarchyInTree, queueInTree.Name)))
                        }</span>
                }
        }
        <span class="cov8" title="1">return errs</span>
}

func validateStateOfQueue(value schedulingv1beta1.QueueState, fldPath *field.Path) field.ErrorList <span class="cov8" title="1">{
        errs := field.ErrorList{}

        if len(value) == 0 </span><span class="cov8" title="1">{
                return errs
        }</span>

        <span class="cov8" title="1">validQueueStates := []schedulingv1beta1.QueueState{
                schedulingv1beta1.QueueStateOpen,
                schedulingv1beta1.QueueStateClosed,
        }

        for _, validQueue := range validQueueStates </span><span class="cov8" title="1">{
                if value == validQueue </span><span class="cov8" title="1">{
                        return errs
                }</span>
        }

        <span class="cov8" title="1">return append(errs, field.Invalid(fldPath, value, fmt.Sprintf("queue state must be in %v", validQueueStates)))</span>
}

func validateWeightOfQueue(value int32, fldPath *field.Path) field.ErrorList <span class="cov8" title="1">{
        errs := field.ErrorList{}
        if value &gt; 0 </span><span class="cov8" title="1">{
                return errs
        }</span>
        <span class="cov8" title="1">return append(errs, field.Invalid(fldPath, value, "queue weight must be a positive integer"))</span>
}

func validateQueueDeleting(queue string) error <span class="cov8" title="1">{
        if queue == "default" </span><span class="cov8" title="1">{
                return fmt.Errorf("`%s` queue can not be deleted", "default")
        }</span>

        <span class="cov8" title="1">q, err := config.VolcanoClient.SchedulingV1beta1().Queues().Get(context.TODO(), queue, metav1.GetOptions{})
        if err != nil </span><span class="cov0" title="0">{
                return err
        }</span>

        <span class="cov8" title="1">if q.Status.State != schedulingv1beta1.QueueStateClosed </span><span class="cov8" title="1">{
                return fmt.Errorf("only queue with state `%s` can be deleted, queue `%s` state is `%s`",
                        schedulingv1beta1.QueueStateClosed, q.Name, q.Status.State)
        }</span>

        <span class="cov8" title="1">return nil</span>
}
</pre>
		
		</div>
	</body>
	<script>
	(function() {
		var files = document.getElementById('files');
		var visible;
		files.addEventListener('change', onChange, false);
		function select(part) {
			if (visible)
				visible.style.display = 'none';
			visible = document.getElementById(part);
			if (!visible)
				return;
			files.value = part;
			visible.style.display = 'block';
			location.hash = part;
		}
		function onChange() {
			select(files.value);
			window.scrollTo(0, 0);
		}
		if (location.hash != "") {
			select(location.hash.substr(1));
		}
		if (!visible) {
			select("file0");
		}
	})();
	</script>
</html>
